You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by wu...@apache.org on 2022/11/28 14:21:59 UTC
[ambari] branch trunk updated: AMBARI-25630: Kerberization of the big cluster using Blueprints takes too much time (#3578)
This is an automated email from the ASF dual-hosted git repository.
wuzhiguo pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new a103423b3e AMBARI-25630: Kerberization of the big cluster using Blueprints takes too much time (#3578)
a103423b3e is described below
commit a103423b3e777ac0f61a5ff3bb6bf06ddd369fa6
Author: Zhiguo Wu <wu...@apache.org>
AuthorDate: Mon Nov 28 22:21:51 2022 +0800
AMBARI-25630: Kerberization of the big cluster using Blueprints takes too much time (#3578)
---
.../ambari/server/AmbariRuntimeException.java | 4 +
.../server/actionmanager/ActionScheduler.java | 4 +-
.../server/agent/stomp/AgentConfigsHolder.java | 10 +-
.../server/agent/stomp/AgentHostDataHolder.java | 8 +-
.../server/agent/stomp/AlertDefinitionsHolder.java | 5 +-
.../ambari/server/configuration/Configuration.java | 24 +-
.../controller/AmbariManagementController.java | 6 +-
.../controller/AmbariManagementControllerImpl.java | 6 +-
.../ambari/server/controller/ControllerModule.java | 5 +
.../server/controller/DeleteIdentityHandler.java | 4 +-
.../ambari/server/controller/KerberosHelper.java | 114 ++++-
.../server/controller/KerberosHelperImpl.java | 543 ++++++++++++---------
.../ClusterKerberosDescriptorResourceProvider.java | 5 +-
.../HostKerberosIdentityResourceProvider.java | 2 +-
.../metrics/RestMetricsPropertyProvider.java | 2 +-
.../events/publishers/AgentCommandsPublisher.java | 131 +++--
.../events/publishers/STOMPUpdatePublisher.java | 21 +-
.../ambari/server/orm/dao/KerberosKeytabDAO.java | 11 +
.../server/orm/dao/KerberosKeytabPrincipalDAO.java | 81 ++-
.../ambari/server/orm/entities/ArtifactEntity.java | 29 +-
.../server/orm/entities/KerberosKeytabEntity.java | 12 +
.../entities/ServiceStackEntity.java} | 26 +-
.../ambari/server/orm/entities/StackEntity.java | 21 +-
.../AbstractPrepareKerberosServerAction.java | 18 +-
.../ConfigureAmbariIdentitiesServerAction.java | 2 +-
.../kerberos/CreateKeytabFilesServerAction.java | 34 +-
.../kerberos/CreatePrincipalsServerAction.java | 1 +
.../kerberos/FinalizeKerberosServerAction.java | 1 +
.../kerberos/KerberosServerAction.java | 101 ++--
.../kerberos/MITKerberosOperationHandler.java | 35 +-
.../PrepareDisableKerberosServerAction.java | 2 +-
.../PrepareEnableKerberosServerAction.java | 2 +-
.../PrepareKerberosIdentitiesServerAction.java | 4 +-
.../UpdateKerberosConfigsServerAction.java | 10 +-
.../stageutils/KerberosKeytabController.java | 48 +-
.../upgrades/PreconfigureKerberosAction.java | 20 +-
.../org/apache/ambari/server/state/Cluster.java | 11 +
.../apache/ambari/server/state/ConfigHelper.java | 149 ++++--
.../java/org/apache/ambari/server/state/Host.java | 4 +-
.../ambari/server/state/cluster/ClusterImpl.java | 31 +-
.../ambari/server/state/cluster/ClustersImpl.java | 5 +-
.../apache/ambari/server/state/host/HostImpl.java | 56 +--
.../state/kerberos/VariableReplacementHelper.java | 37 +-
.../ambari/server/upgrade/UpgradeCatalog270.java | 2 +-
.../apache/ambari/server/utils/ThreadPools.java | 188 +++++++
.../ambari/server/agent/TestHeartbeatHandler.java | 10 +-
.../server/controller/KerberosHelperTest.java | 40 +-
...sterKerberosDescriptorResourceProviderTest.java | 12 +-
.../ClusterStackVersionResourceProviderTest.java | 12 +-
.../HostKerberosIdentityResourceProviderTest.java | 2 +-
.../metrics/RestMetricsPropertyProviderTest.java | 3 +-
.../AbstractPrepareKerberosServerActionTest.java | 2 +
.../ConfigureAmbariIdentitiesServerActionTest.java | 2 +-
.../kerberos/KerberosServerActionTest.java | 2 +-
.../UpdateKerberosConfigsServerActionTest.java | 45 +-
.../upgrades/PreconfigureKerberosActionTest.java | 2 +-
.../kerberos/VariableReplacementHelperTest.java | 5 +-
57 files changed, 1248 insertions(+), 724 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java b/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java
index b26f10634e..ecf7b89281 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java
@@ -29,4 +29,8 @@ public class AmbariRuntimeException extends RuntimeException {
public AmbariRuntimeException(String message) {
super(message);
}
+
+ public AmbariRuntimeException(Throwable cause){
+ super(cause);
+ }
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
index c7cbeacaaa..6c9252d37e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionScheduler.java
@@ -439,9 +439,7 @@ class ActionScheduler implements Runnable {
// Commands that will be scheduled in current scheduler wakeup
List<ExecutionCommand> commandsToSchedule = new ArrayList<>();
Multimap<Long, AgentCommand> commandsToEnqueue = ArrayListMultimap.create();
-
- Map<String, RoleStats> roleStats =
- processInProgressStage(stage, commandsToSchedule, commandsToEnqueue);
+ Map<String, RoleStats> roleStats = processInProgressStage(stage, commandsToSchedule, commandsToEnqueue);
// Check if stage is failed
boolean failed = false;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentConfigsHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentConfigsHolder.java
index 66a08094c2..5845f90da9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentConfigsHolder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentConfigsHolder.java
@@ -17,6 +17,7 @@
*/
package org.apache.ambari.server.agent.stomp;
+import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
@@ -27,6 +28,7 @@ import org.apache.ambari.server.security.encryption.Encryptor;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.utils.ThreadPools;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -48,6 +50,9 @@ public class AgentConfigsHolder extends AgentHostDataHolder<AgentConfigsUpdateEv
@Inject
private Provider<Clusters> clusters;
+ @Inject
+ private ThreadPools threadPools;
+
@Inject
public AgentConfigsHolder(AmbariEventPublisher ambariEventPublisher, @Named("AgentConfigEncryptor") Encryptor<AgentConfigsUpdateEvent> encryptor) {
this.encryptor = encryptor;
@@ -71,10 +76,11 @@ public class AgentConfigsHolder extends AgentHostDataHolder<AgentConfigsUpdateEv
public void updateData(Long clusterId, List<Long> hostIds) throws AmbariException {
if (CollectionUtils.isEmpty(hostIds)) {
// TODO cluster configs will be created before hosts assigning
- if (CollectionUtils.isEmpty(clusters.get().getCluster(clusterId).getHosts())) {
+ Collection<Host> hosts = clusters.get().getCluster(clusterId).getHosts();
+ if (CollectionUtils.isEmpty(hosts)) {
hostIds = clusters.get().getHosts().stream().map(Host::getHostId).collect(Collectors.toList());
} else {
- hostIds = clusters.get().getCluster(clusterId).getHosts().stream().map(Host::getHostId).collect(Collectors.toList());
+ hostIds = hosts.stream().map(Host::getHostId).collect(Collectors.toList());
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentHostDataHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentHostDataHolder.java
index af4ebee8f0..7205d62135 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentHostDataHolder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AgentHostDataHolder.java
@@ -53,12 +53,8 @@ public abstract class AgentHostDataHolder<T extends STOMPHostEvent & Hashable> e
return !Objects.equals(agentHash, hostData.getHash()) ? hostData : getEmptyData();
}
- public T initializeDataIfNeeded(Long hostId, boolean regenerateHash) throws AmbariException {
- try {
- return data.computeIfAbsent(hostId, id -> initializeData(hostId, regenerateHash));
- } catch (AmbariRuntimeException e) {
- throw new AmbariException(e.getMessage(), e);
- }
+ public T initializeDataIfNeeded(Long hostId, boolean regenerateHash) throws AmbariRuntimeException {
+ return data.computeIfAbsent(hostId, id -> initializeData(hostId, regenerateHash));
}
private T initializeData(Long hostId, boolean regenerateHash) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AlertDefinitionsHolder.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AlertDefinitionsHolder.java
index 14b5877784..6f3723f360 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AlertDefinitionsHolder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/AlertDefinitionsHolder.java
@@ -30,6 +30,7 @@ import javax.inject.Provider;
import javax.inject.Singleton;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.agent.stomp.dto.AlertCluster;
import org.apache.ambari.server.events.AlertDefinitionEventType;
import org.apache.ambari.server.events.AlertDefinitionsAgentUpdateEvent;
@@ -189,10 +190,10 @@ public class AlertDefinitionsHolder extends AgentHostDataHolder<AlertDefinitions
}
}
- private void safelyUpdateData(AlertDefinitionsAgentUpdateEvent event) {
+ private void safelyUpdateData(AlertDefinitionsAgentUpdateEvent event) throws AmbariException {
try {
updateData(event);
- } catch (AmbariException e) {
+ } catch (AmbariRuntimeException e) {
LOG.warn(String.format("Failed to %s alert definitions for host %s", event.getEventType(), event.getHostName()), e);
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 825d214875..129df45a79 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -2658,6 +2658,14 @@ public class Configuration {
public static final ConfigurationProperty<Integer> KERBEROS_SERVER_ACTION_THREADPOOL_SIZE = new ConfigurationProperty<>(
"server.kerberos.action.threadpool.size", 1);
+ @Markdown(description = "The Agent command publisher pool. Affects degree of parallelization for generating the commands.")
+ public static final ConfigurationProperty<Integer> AGENT_COMMAND_PUBLISHER_THREADPOOL_SIZE = new ConfigurationProperty<>(
+ "server.pools.agent.command.publisher.size", 5);
+
+ @Markdown(description = "Configures size of the default JOIN Fork pool used for Streams.")
+ public static final ConfigurationProperty<Integer> DEFAULT_FORK_JOIN_THREADPOOL_SIZE = new ConfigurationProperty<>(
+ "server.pools.default.size", 5);
+
/**
* A flag to determine whether error stacks appear on the error page
*/
@@ -5671,10 +5679,24 @@ public class Configuration {
*
* @return the threadpool size, defaulting to 1
*/
- public int getKerberosServerActionThreadpoolSize() {
+ public int getKerberosServerActionThreadPoolSize() {
return Integer.parseInt(getProperty(KERBEROS_SERVER_ACTION_THREADPOOL_SIZE));
}
+ /**
+ * Determines the amount of threads dedicated for {@link org.apache.ambari.server.events.publishers.AgentCommandsPublisher}
+ */
+ public int getAgentCommandPublisherThreadPoolSize() {
+ return Integer.parseInt(getProperty(AGENT_COMMAND_PUBLISHER_THREADPOOL_SIZE));
+ }
+
+ /**
+ * Determines the amount of threads used by default ForJoin Pool
+ */
+ public int getDefaultForkJoinPoolSize(){
+ return Integer.parseInt(getProperty(DEFAULT_FORK_JOIN_THREADPOOL_SIZE));
+ }
+
/**
* Get the timeout, in seconds, when finalizing Kerberos
* enable/disable/regenerate commands.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
index 0e65f51c09..a95e185343 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
@@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.actionmanager.ActionManager;
@@ -56,6 +58,7 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
import org.apache.ambari.server.state.Module;
@@ -775,7 +778,8 @@ public interface AmbariManagementController {
* @throws AmbariException if configuration tags can not be obtained
*/
Map<String, Map<String,String>> findConfigurationTagsWithOverrides(
- Cluster cluster, String hostName) throws AmbariException;
+ Cluster cluster, String hostName,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException;
/**
* Returns parameters for RCA database
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 421bb13027..0d36d69b29 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -86,6 +86,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
import javax.persistence.RollbackException;
import org.apache.ambari.annotations.Experimental;
@@ -2514,9 +2515,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
@Override
public Map<String, Map<String,String>> findConfigurationTagsWithOverrides(
- Cluster cluster, String hostName) throws AmbariException {
+ Cluster cluster, String hostName,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
- return configHelper.getEffectiveDesiredTags(cluster, hostName);
+ return configHelper.getEffectiveDesiredTags(cluster, hostName, desiredConfigs);
}
@Override
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
index 12721fe9ed..38ecfaab2d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
@@ -167,6 +167,7 @@ import org.apache.ambari.server.topology.PersistedStateImpl;
import org.apache.ambari.server.topology.SecurityConfigurationFactory;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
import org.apache.ambari.server.utils.PasswordUtils;
+import org.apache.ambari.server.utils.ThreadPools;
import org.apache.ambari.server.view.ViewInstanceHandlerList;
import org.eclipse.jetty.server.session.SessionHandler;
import org.slf4j.Logger;
@@ -203,6 +204,7 @@ public class ControllerModule extends AbstractModule {
private final Configuration configuration;
private final OsFamily os_family;
private final HostsMap hostsMap;
+ private final ThreadPools threadPools;
private boolean dbInitNeeded;
private final Gson prettyGson = new GsonBuilder().serializeNulls().setPrettyPrinting().create();
@@ -213,12 +215,14 @@ public class ControllerModule extends AbstractModule {
configuration = new Configuration();
hostsMap = new HostsMap(configuration);
os_family = new OsFamily(configuration);
+ threadPools = new ThreadPools(configuration);
}
public ControllerModule(Properties properties) throws Exception {
configuration = new Configuration(properties);
hostsMap = new HostsMap(configuration);
os_family = new OsFamily(configuration);
+ threadPools = new ThreadPools((configuration));
}
@@ -353,6 +357,7 @@ public class ControllerModule extends AbstractModule {
bind(Configuration.class).toInstance(configuration);
bind(OsFamily.class).toInstance(os_family);
+ bind(ThreadPools.class).toInstance(threadPools);
bind(HostsMap.class).toInstance(hostsMap);
bind(PasswordEncoder.class).toInstance(new StandardPasswordEncoder());
bind(DelegatingFilterProxy.class).toInstance(new DelegatingFilterProxy() {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index b9ea37da9a..23abaceb67 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -292,7 +292,7 @@ class DeleteIdentityHandler {
* The service configuration is needed because principal names may contain placeholder variables which are replaced based on the service configuration.
*/
private Map<String, Map<String, String>> calculateConfig(KerberosDescriptor kerberosDescriptor, Set<String> serviceNames) throws AmbariException {
- Map<String, Map<String, String>> actualConfig = getKerberosHelper().calculateConfigurations(getCluster(), null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> actualConfig = getKerberosHelper().calculateConfigurations(getCluster(), null, kerberosDescriptor, false, false, null);
extendWithDeletedConfigOfService(actualConfig, serviceNames);
return actualConfig;
}
@@ -358,7 +358,7 @@ class DeleteIdentityHandler {
Stage stage = createNewStage(id, cluster, requestId, requestContext, commandParams, hostParams);
stage.addServerActionCommand(actionClass.getName(), null, Role.AMBARI_SERVER_ACTION,
RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail,
- ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout,
+ ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null), timeout,
false, false);
return stage;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index 7fe54e6734..a52258e0c1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -25,8 +25,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.internal.RequestStageContainer;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.serveraction.kerberos.Component;
import org.apache.ambari.server.serveraction.kerberos.KerberosAdminAuthenticationException;
@@ -36,6 +39,7 @@ import org.apache.ambari.server.serveraction.kerberos.KerberosMissingAdminCreden
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.StackId;
@@ -507,10 +511,38 @@ public interface KerberosHelper {
* @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
* @return the kerberos descriptor associated with the specified cluster
* @throws AmbariException if unable to obtain the descriptor
- * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection, boolean)
+ * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection, boolean, KerberosDescriptor, Map)
*/
KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData) throws AmbariException;
+ /**
+ * Builds a composite Kerberos descriptor using the default Kerberos descriptor and a user-specified
+ * Kerberos descriptor, if it exists.
+ * <p/>
+ * The default Kerberos descriptor is built from the kerberos.json files in the stack. It can be
+ * retrieved via the <code>stacks/:stackName/versions/:version/artifacts/kerberos_descriptor</code>
+ * endpoint
+ * <p/>
+ * The user-specified Kerberos descriptor was registered to the
+ * <code>cluster/:clusterName/artifacts/kerberos_descriptor</code> endpoint.
+ * <p/>
+ * If the user-specified Kerberos descriptor exists, it is used to update the default Kerberos
+ * descriptor and the composite is returned. If not, the default cluster descriptor is returned
+ * as-is.
+ *
+ * @param cluster cluster instance
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
+ * @param userDescriptor user defined Kerberos descriptor properties
+ * @param desiredConfigs desired config map
+ * @return the kerberos descriptor associated with the specified cluster
+ * @throws AmbariException if unable to obtain the descriptor
+ * @see #getKerberosDescriptor(KerberosDescriptorType, Cluster, boolean, Collection, boolean, KerberosDescriptor, Map)
+ */
+ KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException;
+
/**
* Gets the requested Kerberos descriptor.
* <p>
@@ -534,11 +566,16 @@ public interface KerberosHelper {
* services to add to the set of currently installed services to use
* while evaluating <code>when</code> clauses
* @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
+ * @param userDescriptor user defined Kerberos descriptor properties
+ * @param desiredConfigs desired config mao
* @return a Kerberos descriptor
* @throws AmbariException
*/
KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
- boolean evaluateWhenClauses, Collection<String> additionalServices, boolean includePreconfigureData)
+ boolean evaluateWhenClauses, Collection<String> additionalServices,
+ boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException;
/**
@@ -559,10 +596,13 @@ public interface KerberosHelper {
* @param cluster the relevant Cluster
* @param stackId the relevant stack id, used for <code>COMPOSITE</code> or <code>STACK</code> Kerberos descriptor requests
* @param includePreconfigureData <code>true</code> to include the preconfigure data; <code>false</code> otherwise
+ * @param userDescriptor user defined Kerberos descriptor properties
* @return a Kerberos descriptor
* @throws AmbariException
*/
- KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster, StackId stackId, boolean includePreconfigureData)
+ KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
+ StackId stackId, boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor)
throws AmbariException;
/**
@@ -642,13 +682,63 @@ public interface KerberosHelper {
* @param kerberosDescriptor a map of general Kerberos descriptor properties
* @param includePreconfigureData <code>true</code> to include the preconfigure data; otherwise false
* @param calculateClusterHostInfo
+ * @param componentHosts map of cached cluster host info
+ * @param desiredConfigs desired configuration map
* @return a Map of calculated configuration types
* @throws AmbariException
*/
Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
KerberosDescriptor kerberosDescriptor,
boolean includePreconfigureData,
- boolean calculateClusterHostInfo)
+ boolean calculateClusterHostInfo,
+ Map<String, String> componentHosts,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException;
+
+ /**
+ * Calculates the map of configurations relative to the cluster and host.
+ * <p/>
+ * Most of this was borrowed from {@link org.apache.ambari.server.actionmanager.ExecutionCommandWrapper#getExecutionCommand()}
+ *
+ * @param cluster the relevant Cluster
+ * @param hostname the relevant hostname
+ * @param kerberosDescriptor a map of general Kerberos descriptor properties
+ * @param userDescriptor a map of user defined Kerberos descriptor properties
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; otherwise false
+ * @param calculateClusterHostInfo
+ * @param componentHosts map of cached cluster host info
+ * @param desiredConfigs desired configuration map
+ * @return a Map of calculated configuration types
+ * @throws AmbariException
+ */
+ Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
+ KerberosDescriptor kerberosDescriptor,
+ KerberosDescriptor userDescriptor,
+ boolean includePreconfigureData,
+ boolean calculateClusterHostInfo,
+ Map<String, String> componentHosts,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException;
+
+ /**
+ * Calculates the map of configurations relative to the cluster and host.
+ * <p/>
+ * Most of this was borrowed from {@link org.apache.ambari.server.actionmanager.ExecutionCommandWrapper#getExecutionCommand()}
+ *
+ * @param cluster the relevant Cluster
+ * @param hostname the relevant hostname
+ * @param kerberosDescriptor a map of general Kerberos descriptor properties
+ * @param includePreconfigureData <code>true</code> to include the preconfigure data; otherwise false
+ * @param calculateClusterHostInfo
+ * @param desiredConfigs desired configuration map
+ * @return a Map of calculated configuration types
+ * @throws AmbariException
+ */
+ Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
+ KerberosDescriptor kerberosDescriptor,
+ boolean includePreconfigureData,
+ boolean calculateClusterHostInfo,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException;
/**
@@ -748,6 +838,7 @@ public interface KerberosHelper {
* the kerberos descriptor to use when looking up identities. If
* {@code null}, then this method will deserialize the descriptor
* inside of a loop at considerable cost.
+ * @param desiredConfigs desired configuration map
* @return a map of host names to kerberos identities
* @throws AmbariException
* if an error occurs processing the cluster's active identities
@@ -758,7 +849,8 @@ public interface KerberosHelper {
String componentName,
boolean replaceHostNames,
Map<String, Map<String, Map<String, String>>> hostConfigurations,
- KerberosDescriptor kerberosDescriptor)
+ KerberosDescriptor kerberosDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException;
/**
@@ -792,8 +884,10 @@ public interface KerberosHelper {
* Saves underlying entities in persistent storage.
*
* @param resolvedKerberosKeytab kerberos keytab to be persisted
+ * @param keytabList all available keytab
*/
- void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab);
+ void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab,
+ List<KerberosKeytabPrincipalEntity> keytabList);
/**
* Removes existent persisted keytabs if they are not in {@code expectedKeytabs} collection.
@@ -858,6 +952,14 @@ public interface KerberosHelper {
KerberosDetails getKerberosDetails(Cluster cluster, Boolean manageIdentities)
throws KerberosInvalidConfigurationException, AmbariException;
+ /**
+ * Get the user-supplied Kerberos descriptor from the set of cluster artifacts
+ *
+ * @param cluster the cluster
+ * @return a Kerberos descriptor
+ */
+ KerberosDescriptor getKerberosDescriptorUpdates(Cluster cluster);
+
/**
* Types of Kerberos descriptors related to where the data is stored.
* <dl>
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index b8b613ba5c..9a40938ada 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -40,6 +40,8 @@ import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
+import javax.annotation.Nullable;
+
import org.apache.ambari.annotations.Experimental;
import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
@@ -108,6 +110,7 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.PropertyInfo;
@@ -439,40 +442,33 @@ public class KerberosHelperImpl implements KerberosHelper {
// We can create the map in the "shouldIncludeCommand" Command to avoid having to iterate
// over the returned ServiceComponentHost List.
getServiceComponentHosts(cluster,
- new Command<Boolean, ServiceComponentHost>() {
- @Override
- public Boolean invoke(ServiceComponentHost sch) throws AmbariException {
- if (sch != null) {
- String serviceName = sch.getServiceName();
+ sch -> {
+ if (sch != null) {
+ String serviceName = sch.getServiceName();
- Set<String> installedComponents = installedServices.get(serviceName);
- if (installedComponents == null) {
- installedComponents = new HashSet<>();
- installedServices.put(serviceName, installedComponents);
- }
- installedComponents.add(sch.getServiceComponentName());
+ Set<String> installedComponents = installedServices.computeIfAbsent(serviceName, k -> new HashSet<>());
+ installedComponents.add(sch.getServiceComponentName());
- // Determine if this component was PREVIOUSLY installed, which implies that its containing service was PREVIOUSLY installed
- if (!previouslyExistingServices.contains(serviceName) && PREVIOUSLY_INSTALLED_STATES.contains(sch.getState())) {
- previouslyExistingServices.add(serviceName);
- }
-
- return true;
+ // Determine if this component was PREVIOUSLY installed, which implies that its containing service was PREVIOUSLY installed
+ if (!previouslyExistingServices.contains(serviceName) && PREVIOUSLY_INSTALLED_STATES.contains(sch.getState())) {
+ previouslyExistingServices.add(serviceName);
}
- return false;
+ return true;
}
+
+ return false;
});
- Map<String, Map<String, String>> existingConfigurations = configHelper.calculateExistingConfigurations(ambariManagementController, cluster, null);
+ Map<String, Map<String, String>> existingConfigurations = configHelper.calculateExistingConfigurations(ambariManagementController, cluster, null, null);
Map<String, Map<String, String>> updates = getServiceConfigurationUpdates(cluster,
existingConfigurations, installedServices, serviceFilter, previouslyExistingServices, true, true);
// Store the updates...
for (Map.Entry<String, Map<String, String>> entry : updates.entrySet()) {
- configHelper.updateConfigType(cluster, cluster.getDesiredStackVersion(),
- ambariManagementController, entry.getKey(), entry.getValue(), null,
- ambariManagementController.getAuthName(), "Enabling Kerberos for added components");
+ configHelper.updateConfigType(cluster, cluster.getDesiredStackVersion(), ambariManagementController,
+ entry.getKey(), entry.getValue(), null, ambariManagementController.getAuthName(),
+ "Enabling Kerberos for added components");
}
}
@@ -492,7 +488,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
- deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+ deepCopy(existingConfigurations), null, kerberosDescriptorProperties, null);
Map<String, Set<String>> propertiesToIgnore = new HashMap<>();
@@ -896,7 +892,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
- deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+ deepCopy(existingConfigurations), null, kerberosDescriptorProperties, null);
Map<String, String> kerberosConfiguration = kerberosDetails.getKerberosEnvProperties();
KerberosOperationHandler kerberosOperationHandler = kerberosOperationHandlerFactory.getKerberosOperationHandler(kerberosDetails.getKdcType());
@@ -1371,31 +1367,32 @@ public class KerberosHelperImpl implements KerberosHelper {
return hostsWithValidKerberosClient;
}
+ @Override
+ public KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
+ return getKerberosDescriptor(KerberosDescriptorType.COMPOSITE, cluster, false, null,
+ includePreconfigureData, userDescriptor, desiredConfigs);
+ }
+
@Override
public KerberosDescriptor getKerberosDescriptor(Cluster cluster, boolean includePreconfigureData) throws AmbariException {
- return getKerberosDescriptor(KerberosDescriptorType.COMPOSITE, cluster, false, null, includePreconfigureData);
+ return getKerberosDescriptor(KerberosDescriptorType.COMPOSITE, cluster, false, null,
+ includePreconfigureData, null, null);
}
@Override
public KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
boolean evaluateWhenClauses, Collection<String> additionalServices,
- boolean includePreconfigureData)
+ boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException {
- // !!! FIXME in a per-service view, what does this become?
- Set<StackId> stackIds = new HashSet<>();
-
- for (Service service : cluster.getServices().values()) {
- stackIds.add(service.getDesiredStackId());
- }
-
- if (1 != stackIds.size()) {
- throw new AmbariException("Services are deployed from multiple stacks and cannot determine a unique one.");
- }
-
- StackId stackId = stackIds.iterator().next();
-
- KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(kerberosDescriptorType, cluster, stackId, includePreconfigureData);
+ // Checking stackId for each service is too much expensive, assume that all services following the cluster desired setting
+ StackId stackId = cluster.getDesiredStackVersion();
+ KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(kerberosDescriptorType, cluster,
+ stackId, includePreconfigureData, userDescriptor);
if (evaluateWhenClauses) {
Set<String> services = new HashSet<>(cluster.getServices().keySet());
@@ -1407,7 +1404,8 @@ public class KerberosHelperImpl implements KerberosHelper {
// Build the context needed to filter out Kerberos identities...
// This includes the current set of configurations for the cluster and the set of installed services
Map<String, Object> context = new HashMap<>();
- context.put("configurations", calculateConfigurations(cluster, null, kerberosDescriptor, false, false));
+ context.put("configurations", calculateConfigurations(cluster, null, kerberosDescriptor,
+ false, false, desiredConfigs));
context.put("services", services);
// Get the Kerberos identities that need to be pruned
@@ -1445,16 +1443,20 @@ public class KerberosHelperImpl implements KerberosHelper {
@Override
public KerberosDescriptor getKerberosDescriptor(KerberosDescriptorType kerberosDescriptorType, Cluster cluster,
- StackId stackId, boolean includePreconfigureData) throws AmbariException {
+ StackId stackId, boolean includePreconfigureData,
+ @Nullable KerberosDescriptor userDescriptor) throws AmbariException {
+
KerberosDescriptor stackDescriptor = (kerberosDescriptorType == KerberosDescriptorType.STACK || kerberosDescriptorType == KerberosDescriptorType.COMPOSITE)
? getKerberosDescriptorFromStack(stackId, includePreconfigureData)
: null;
- KerberosDescriptor userDescriptor = (kerberosDescriptorType == KerberosDescriptorType.USER || kerberosDescriptorType == KerberosDescriptorType.COMPOSITE)
+ KerberosDescriptor finalUserDescriptor = (kerberosDescriptorType == KerberosDescriptorType.USER || kerberosDescriptorType == KerberosDescriptorType.COMPOSITE)
+ ? (userDescriptor == null)
? getKerberosDescriptorUpdates(cluster)
+ : userDescriptor
: null;
- return combineKerberosDescriptors(stackDescriptor, userDescriptor);
+ return combineKerberosDescriptors(stackDescriptor, finalUserDescriptor);
}
@Override
@@ -1713,27 +1715,68 @@ public class KerberosHelperImpl implements KerberosHelper {
public Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
KerberosDescriptor kerberosDescriptor,
boolean includePreconfigureData,
- boolean calculateClusterHostInfo)
+ boolean calculateClusterHostInfo,
+ Map<String, String> componentHosts,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException {
+
+ return calculateConfigurations(cluster, hostname, kerberosDescriptor, null,
+ includePreconfigureData, calculateClusterHostInfo, componentHosts, desiredConfigs);
+ }
+
+ @Override
+ public Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
+ KerberosDescriptor kerberosDescriptor,
+ KerberosDescriptor userDescriptor,
+ boolean includePreconfigureData,
+ boolean calculateClusterHostInfo,
+ Map<String, String> componentHosts,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException {
Map<String, Map<String, String>> calculatedConfigurations = addAdditionalConfigurations(
cluster,
- configHelper.calculateExistingConfigurations(ambariManagementController, cluster, hostname),
+ configHelper.calculateExistingConfigurations(ambariManagementController, cluster, hostname, desiredConfigs),
hostname,
- (kerberosDescriptor == null) ? null : kerberosDescriptor.getProperties());
+ (kerberosDescriptor == null) ? null : kerberosDescriptor.getProperties(),
+ userDescriptor,
+ componentHosts,
+ desiredConfigs
+ );
if (includePreconfigureData) {
- calculatedConfigurations = addConfigurationsForPreProcessedServices(calculatedConfigurations, cluster, kerberosDescriptor, calculateClusterHostInfo);
+ calculatedConfigurations = addConfigurationsForPreProcessedServices(
+ calculatedConfigurations,
+ cluster,
+ kerberosDescriptor,
+ calculateClusterHostInfo
+ );
}
return calculatedConfigurations;
}
- private Map<String, String> principalNames(Cluster cluster, Map<String, Map<String, String>> configuration) throws AmbariException {
+ @Override
+ public Map<String, Map<String, String>> calculateConfigurations(Cluster cluster, String hostname,
+ KerberosDescriptor kerberosDescriptor,
+ boolean includePreconfigureData,
+ boolean calculateClusterHostInfo,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException {
+ return calculateConfigurations(cluster, hostname, kerberosDescriptor, includePreconfigureData,
+ calculateClusterHostInfo, null, desiredConfigs);
+ }
+
+ private Map<String, String> principalNames(Cluster cluster, Map<String, Map<String, String>> configuration,
+ @Nullable KerberosDescriptor userDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException {
+
Map<String, String> result = new HashMap<>();
- for (Map.Entry<String, String> each : getKerberosDescriptor(cluster, false).principals().entrySet()) {
- result.put(each.getKey(), variableReplacementHelper.replaceVariables(each.getValue(), configuration));
- }
+ getKerberosDescriptor(cluster, false, userDescriptor, desiredConfigs).principals()
+ .forEach((key, value) -> {
+ result.put(key, variableReplacementHelper.replaceVariables(value, configuration));
+ });
return result;
}
@@ -1744,7 +1787,8 @@ public class KerberosHelperImpl implements KerberosHelper {
String componentName,
boolean replaceHostNames,
Map<String, Map<String, Map<String, String>>> hostConfigurations,
- KerberosDescriptor kerberosDescriptor)
+ KerberosDescriptor kerberosDescriptor,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
throws AmbariException {
if ((clusterName == null) || clusterName.isEmpty()) {
@@ -1759,153 +1803,153 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Collection<KerberosIdentityDescriptor>> activeIdentities = new HashMap<>();
- // Only calculate the active identities if the kerberos-env configurtaion is available. Else
+ // Only calculate the active identities if the kerberos-env configuration is available. Else
// important information like the realm will be missing (kerberos-env/realm)
- Config kerberosEnvConfig = cluster.getDesiredConfigByType(KERBEROS_ENV);
+ Config kerberosEnvConfig = cluster.getDesiredConfigByType(KERBEROS_ENV, desiredConfigs);
if (kerberosEnvConfig == null) {
LOG.debug("Calculating the active identities for {} is being skipped since the kerberos-env configuration is not available",
clusterName, cluster.getSecurityType().name(), SecurityType.KERBEROS.name());
- } else {
- Collection<String> hosts;
- String ambariServerHostname = StageUtils.getHostName();
- boolean ambariServerHostnameIsForced = false;
-
- if (hostName == null) {
- Map<String, Host> hostMap = clusters.getHostsForCluster(clusterName);
- if (hostMap == null) {
- hosts = Collections.emptySet();
- } else {
- hosts = hostMap.keySet();
- }
+ return activeIdentities;
+ }
- if (!hosts.contains(ambariServerHostname)) {
- Collection<String> extendedHosts = new ArrayList<>(hosts.size() + 1);
- extendedHosts.addAll(hosts);
- extendedHosts.add(ambariServerHostname);
- hosts = extendedHosts;
- ambariServerHostnameIsForced = true;
- }
- } else {
- hosts = Collections.singleton(hostName);
+ Collection<String> hosts;
+ String ambariServerHostname = StageUtils.getHostName();
+ boolean ambariServerHostnameIsForced = false;
+
+ if (hostName == null) {
+ Map<String, Host> hostMap = clusters.getHostsForCluster(clusterName);
+ hosts = (hostMap == null) ? Collections.emptySet() : hostMap.keySet();
+
+ if (!hosts.contains(ambariServerHostname)) {
+ Collection<String> extendedHosts = new ArrayList<>(hosts.size() + 1);
+ extendedHosts.addAll(hosts);
+ extendedHosts.add(ambariServerHostname);
+ hosts = extendedHosts;
+ ambariServerHostnameIsForced = true;
}
+ } else {
+ hosts = Collections.singleton(hostName);
+ }
- if (null == hostConfigurations) {
- hostConfigurations = new HashMap<>();
- }
+ if (null == hostConfigurations) {
+ hostConfigurations = new HashMap<>();
+ }
- if (!hosts.isEmpty()) {
+ if (hosts.isEmpty()) {
+ return activeIdentities;
+ }
- if (null == kerberosDescriptor) {
- kerberosDescriptor = getKerberosDescriptor(cluster, false);
- }
+ if (null == kerberosDescriptor) {
+ kerberosDescriptor = getKerberosDescriptor(cluster, false);
+ }
- if (kerberosDescriptor != null) {
- Set<String> existingServices = cluster.getServices().keySet();
+ if (kerberosDescriptor == null) {
+ return activeIdentities;
+ }
+ Set<String> existingServices = cluster.getServices().keySet();
- for (String host : hosts) {
- // Calculate the current host-specific configurations. These will be used to replace
- // variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = hostConfigurations.get(host);
- if (configurations == null) {
- configurations = calculateConfigurations(cluster,
+ for (String host : hosts) {
+ // Calculate the current host-specific configurations. These will be used to replace
+ // variables within the Kerberos descriptor data
+ Map<String, Map<String, String>> configurations = hostConfigurations.get(host);
+ if (configurations == null) {
+ configurations = calculateConfigurations(cluster,
(ambariServerHostnameIsForced && ambariServerHostname.equals(host)) ? null : host,
kerberosDescriptor,
false,
- false);
+ false,
+ desiredConfigs);
- hostConfigurations.put(host, configurations);
- }
+ hostConfigurations.put(host, configurations);
+ }
- // Create the context to use for filtering Kerberos Identities based on the state of the cluster
- Map<String, Object> filterContext = new HashMap<>();
- filterContext.put("configurations", configurations);
- filterContext.put("services", existingServices);
+ // Create the context to use for filtering Kerberos Identities based on the state of the cluster
+ Map<String, Object> filterContext = new HashMap<>();
+ filterContext.put("configurations", configurations);
+ filterContext.put("services", existingServices);
+ Map<String, KerberosIdentityDescriptor> hostActiveIdentities = new HashMap<>();
+ List<KerberosIdentityDescriptor> identities = getActiveIdentities(cluster, host,
+ serviceName, componentName, kerberosDescriptor, filterContext);
- Map<String, KerberosIdentityDescriptor> hostActiveIdentities = new HashMap<>();
- List<KerberosIdentityDescriptor> identities = getActiveIdentities(cluster, host,
- serviceName, componentName, kerberosDescriptor, filterContext);
- if (host.equals(ambariServerHostname)) {
- // Determine if we should _calculate_ the Ambari service identities.
- // If kerberos-env/create_ambari_principal is not set to false the identity should be calculated.
- if (createAmbariIdentities(kerberosEnvConfig.getProperties())) {
- List<KerberosIdentityDescriptor> ambariIdentities = getAmbariServerIdentities(kerberosDescriptor);
- if (ambariIdentities != null) {
- identities.addAll(ambariIdentities);
- }
- }
- }
+ if (host.equals(ambariServerHostname)) {
+ // Determine if we should _calculate_ the Ambari service identities.
+ // If kerberos-env/create_ambari_principal is not set to false the identity should be calculated.
+ if (createAmbariIdentities(kerberosEnvConfig.getProperties())) {
+ List<KerberosIdentityDescriptor> ambariIdentities = getAmbariServerIdentities(kerberosDescriptor);
+ if (ambariIdentities != null) {
+ identities.addAll(ambariIdentities);
+ }
+ }
+ }
- if (!identities.isEmpty()) {
- for (KerberosIdentityDescriptor identity : identities) {
- KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor();
- String principal = null;
+ if (!identities.isEmpty()) {
+ for (KerberosIdentityDescriptor identity : identities) {
+ KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor();
+ String principal = null;
- if (principalDescriptor != null) {
- principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
- }
+ if (principalDescriptor != null) {
+ principal = variableReplacementHelper.replaceVariables(principalDescriptor.getValue(), configurations);
+ }
- if (principal != null) {
- KerberosKeytabDescriptor keytabDescriptor = identity.getKeytabDescriptor();
- String keytabFile = null;
+ if (principal != null) {
+ KerberosKeytabDescriptor keytabDescriptor = identity.getKeytabDescriptor();
+ String keytabFile = null;
- if (keytabDescriptor != null) {
- keytabFile = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
- }
+ if (keytabDescriptor != null) {
+ keytabFile = variableReplacementHelper.replaceVariables(keytabDescriptor.getFile(), configurations);
+ }
- if (replaceHostNames) {
- principal = principal.replace("_HOST", host);
- }
+ if (replaceHostNames) {
+ principal = principal.replace("_HOST", host);
+ }
- String uniqueKey = String.format("%s|%s", principal, (keytabFile == null) ? "" : keytabFile);
+ String uniqueKey = String.format("%s|%s", principal, (keytabFile == null) ? "" : keytabFile);
- if (!hostActiveIdentities.containsKey(uniqueKey) ||
- (StringUtils.isNotBlank(hostActiveIdentities.get(uniqueKey).getReference()) && StringUtils.isBlank(identity.getReference()))) {
- KerberosPrincipalType principalType = principalDescriptor.getType();
+ if (!hostActiveIdentities.containsKey(uniqueKey) ||
+ (StringUtils.isNotBlank(hostActiveIdentities.get(uniqueKey).getReference()) && StringUtils.isBlank(identity.getReference()))) {
+ KerberosPrincipalType principalType = principalDescriptor.getType();
- // Assume the principal is a service principal if not specified
- if (principalType == null) {
- principalType = KerberosPrincipalType.SERVICE;
- }
+ // Assume the principal is a service principal if not specified
+ if (principalType == null) {
+ principalType = KerberosPrincipalType.SERVICE;
+ }
- KerberosPrincipalDescriptor resolvedPrincipalDescriptor =
- new KerberosPrincipalDescriptor(principal,
- principalType,
- variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
- variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
+ KerberosPrincipalDescriptor resolvedPrincipalDescriptor =
+ new KerberosPrincipalDescriptor(principal,
+ principalType,
+ variableReplacementHelper.replaceVariables(principalDescriptor.getConfiguration(), configurations),
+ variableReplacementHelper.replaceVariables(principalDescriptor.getLocalUsername(), configurations));
- KerberosKeytabDescriptor resolvedKeytabDescriptor;
+ KerberosKeytabDescriptor resolvedKeytabDescriptor;
- if (keytabFile == null) {
- resolvedKeytabDescriptor = null;
- } else {
- resolvedKeytabDescriptor =
+ if (keytabFile == null) {
+ resolvedKeytabDescriptor = null;
+ } else {
+ resolvedKeytabDescriptor =
new KerberosKeytabDescriptor(
- keytabFile,
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
- variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
- keytabDescriptor.isCachable());
- }
+ keytabFile,
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getOwnerAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupName(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getGroupAccess(), configurations),
+ variableReplacementHelper.replaceVariables(keytabDescriptor.getConfiguration(), configurations),
+ keytabDescriptor.isCachable());
+ }
- hostActiveIdentities.put(uniqueKey, new KerberosIdentityDescriptor(
+ hostActiveIdentities.put(uniqueKey, new KerberosIdentityDescriptor(
identity.getName(),
identity.getReference(),
resolvedPrincipalDescriptor,
resolvedKeytabDescriptor,
identity.getWhen()));
- }
- }
- }
}
-
- activeIdentities.put(host, hostActiveIdentities.values());
}
}
}
+
+ activeIdentities.put(host, hostActiveIdentities.values());
}
return activeIdentities;
@@ -1973,60 +2017,50 @@ public class KerberosHelperImpl implements KerberosHelper {
* a {@link KerberosKeytabPrincipalEntity} will result in major performance
* problems.
*
- * @param resolvedKerberosKeytab
- * kerberos keytab to be persisted
+ * @param resolvedKerberosKeytab kerberos keytab to be persisted
+ * @param keytabList cache of existing keytab
*/
- @Override
- public void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab) {
+ @Transactional
+ public void createResolvedKeytab(ResolvedKerberosKeytab resolvedKerberosKeytab,
+ @Nullable List<KerberosKeytabPrincipalEntity> keytabList) {
Stopwatch stopwatch = Stopwatch.createStarted();
+ KerberosKeytabEntity kke = kerberosKeytabDAO.findOrCreate(resolvedKerberosKeytab);
- KerberosKeytabEntity kke = kerberosKeytabDAO.find(resolvedKerberosKeytab.getFile());
- if (null == kke) {
- kke = new KerberosKeytabEntity(resolvedKerberosKeytab.getFile());
- kke.setAmbariServerKeytab(resolvedKerberosKeytab.isAmbariServerKeytab());
- kke.setWriteAmbariJaasFile(resolvedKerberosKeytab.isMustWriteAmbariJaasFile());
- kke.setOwnerName(resolvedKerberosKeytab.getOwnerName());
- kke.setOwnerAccess(resolvedKerberosKeytab.getOwnerAccess());
- kke.setGroupName(resolvedKerberosKeytab.getGroupName());
- kke.setGroupAccess(resolvedKerberosKeytab.getGroupAccess());
- kerberosKeytabDAO.create(kke);
- }
-
- for (ResolvedKerberosPrincipal principal : resolvedKerberosKeytab.getPrincipals()) {
+ resolvedKerberosKeytab.getPrincipals().forEach(principal -> {
KerberosPrincipalEntity kpe = kerberosPrincipalDAO.find(principal.getPrincipal());
- if (null == kpe) {
+ if (kpe == null) {
kpe = kerberosPrincipalDAO.create(principal.getPrincipal(), principal.isService());
}
// only need to merge the kke and kpe if a new kkp is created/added to their lists
- boolean mergeBidirectionalAssociatedEntities = false;
- for (Map.Entry<String, String> mappingEntry : principal.getServiceMapping().entries()) {
- String serviceName = mappingEntry.getKey();
+ final Boolean[] mergeBidirectionalAssociatedEntities = {false};
+ KerberosPrincipalEntity finalKpe = kpe;
+ principal.getServiceMapping().forEach((serviceName, value) -> {
HostEntity hostEntity = principal.getHostId() != null ? hostDAO.findById(principal.getHostId()) : null;
- KeytabPrincipalFindOrCreateResult result = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, kpe);
+ KeytabPrincipalFindOrCreateResult result = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, finalKpe, keytabList);
KerberosKeytabPrincipalEntity kkp = result.kkp;
- mergeBidirectionalAssociatedEntities = mergeBidirectionalAssociatedEntities || result.created;
+ if (keytabList != null && result.created) {
+ keytabList.add(result.kkp);
+ }
+ mergeBidirectionalAssociatedEntities[0] = mergeBidirectionalAssociatedEntities[0] || result.created;
// updating the kkp service mappings does not affect kke/kpe bidirectional relationships
- if (kkp.putServiceMapping(serviceName, mappingEntry.getValue())) {
+ if (kkp.putServiceMapping(serviceName, value)) {
kerberosKeytabPrincipalDAO.merge(kkp);
}
- }
+ });
- // merge the keytab and the principal IFF at least one keytabprincipal was
+ // merge the keytab and the principal IFF at least one keytab principal was
// created causing the bi-directional lists associations to need updating
- if(mergeBidirectionalAssociatedEntities) {
- Stopwatch mergeStockwatch = Stopwatch.createStarted();
- kke = kerberosKeytabDAO.merge(kke);
- kpe = kerberosPrincipalDAO.merge(kpe);
- LOG.info("Merging bidirectional associated entities for this keytab took {}ms"
- + mergeStockwatch.elapsed(TimeUnit.MILLISECONDS));
+ if(mergeBidirectionalAssociatedEntities[0]) {
+ kerberosKeytabDAO.merge(kke);
+ kerberosPrincipalDAO.merge(kpe);
}
- }
+ });
- LOG.info("Resolving this keytab and all associated principals took {}ms ",
+ LOG.info("Resolving this keytab and all associated principals took {} ms ",
stopwatch.elapsed(TimeUnit.MILLISECONDS));
}
@@ -2120,8 +2154,10 @@ public class KerberosHelperImpl implements KerberosHelper {
KerberosKeytabDescriptor keytabDescriptor = identityDescriptor.getKeytabDescriptor();
if (keytabDescriptor != null) {
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal);
keytab = injector.getInstance(CreateKeytabFilesServerAction.class).createKeytab(
principal,
+ principalEntity,
result.getPassword(),
result.getKeyNumber(),
kerberosOperationHandler,
@@ -2389,7 +2425,8 @@ public class KerberosHelperImpl implements KerberosHelper {
// Calculate the current non-host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> configurations = calculateConfigurations(cluster, null,
+ kerberosDescriptor, false, false, null);
String principal = variableReplacementHelper.replaceVariables("${kerberos-env/service_check_principal_name}@${realm}", configurations);
@@ -2435,7 +2472,7 @@ public class KerberosHelperImpl implements KerberosHelper {
}
KeytabPrincipalFindOrCreateResult result = kerberosKeytabPrincipalDAO.findOrCreate(
- kke, hostDAO.findById(sch.getHost().getHostId()), kpe);
+ kke, hostDAO.findById(sch.getHost().getHostId()), kpe, null);
KerberosKeytabPrincipalEntity kkp = result.kkp;
if (kkp.putServiceMapping(sch.getServiceName(), sch.getServiceComponentName())) {
@@ -2923,40 +2960,53 @@ public class KerberosHelperImpl implements KerberosHelper {
String serviceName,
String componentName,
KerberosDescriptor kerberosDescriptor,
- Map<String, Object> filterContext)
- throws AmbariException {
+ Map<String, Object> filterContext) {
List<KerberosIdentityDescriptor> identities = new ArrayList<>();
List<ServiceComponentHost> serviceComponentHosts = cluster.getServiceComponentHosts(hostname);
- if (serviceComponentHosts != null) {
- for (ServiceComponentHost serviceComponentHost : serviceComponentHosts) {
- String schServiceName = serviceComponentHost.getServiceName();
- String schComponentName = serviceComponentHost.getServiceComponentName();
+ if (serviceComponentHosts == null) {
+ return identities;
+ }
+
+ serviceComponentHosts.forEach(serviceComponentHost -> {
+ String schServiceName = serviceComponentHost.getServiceName();
+ String schComponentName = serviceComponentHost.getServiceComponentName();
- if (((serviceName == null) || serviceName.equals(schServiceName)) &&
- ((componentName == null) || componentName.equals(schComponentName))) {
+ if (((serviceName == null) || serviceName.equals(schServiceName)) &&
+ ((componentName == null) || componentName.equals(schComponentName))) {
- KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(schServiceName);
+ KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(schServiceName);
- if (serviceDescriptor != null) {
- List<KerberosIdentityDescriptor> serviceIdentities = serviceDescriptor.getIdentities(true, filterContext);
- if (serviceIdentities != null) {
- identities.addAll(serviceIdentities);
+ if (serviceDescriptor != null) {
+
+ List<KerberosIdentityDescriptor> serviceIdentities;
+ try {
+ serviceIdentities = serviceDescriptor.getIdentities(true, filterContext);
+ } catch (AmbariException e) {
+ serviceIdentities = null;
+ }
+ if (serviceIdentities != null) {
+ identities.addAll(serviceIdentities);
+ }
+
+ KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(schComponentName);
+ if (componentDescriptor != null) {
+ List<KerberosIdentityDescriptor> componentIdentities;
+ try {
+ componentIdentities = componentDescriptor.getIdentities(true, filterContext);
+ } catch (AmbariException e) {
+ componentIdentities = null;
}
- KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(schComponentName);
- if (componentDescriptor != null) {
- List<KerberosIdentityDescriptor> componentIdentities = componentDescriptor.getIdentities(true, filterContext);
- if (componentIdentities != null) {
- identities.addAll(componentIdentities);
- }
+ if (componentIdentities != null) {
+ identities.addAll(componentIdentities);
}
}
}
}
- }
+ });
return identities;
}
@@ -2974,16 +3024,16 @@ public class KerberosHelperImpl implements KerberosHelper {
* @throws AmbariException
*/
private Map<String, Map<String, String>> addAdditionalConfigurations(Cluster cluster, Map<String, Map<String, String>> configurations,
- String hostname, Map<String, String> kerberosDescriptorProperties)
+ String hostname, Map<String, String> kerberosDescriptorProperties,
+ @Nullable KerberosDescriptor userDescriptor,
+ Map<String, String> componentHosts,
+ @Nullable Map<String, DesiredConfig> desiredConfigs
+ )
throws AmbariException {
// A map to hold un-categorized properties. This may come from the KerberosDescriptor
// and will also contain a value for the current host
- Map<String, String> generalProperties = configurations.get("");
- if (generalProperties == null) {
- generalProperties = new HashMap<>();
- configurations.put("", generalProperties);
- }
+ Map<String, String> generalProperties = configurations.computeIfAbsent("", k -> new HashMap<>());
// If any properties are set in the calculated KerberosDescriptor, add them into the
// Map of configurations as an un-categorized type (using an empty string)
@@ -3005,22 +3055,49 @@ public class KerberosHelperImpl implements KerberosHelper {
// add clusterHostInfo config
if (configurations.get(CLUSTER_HOST_INFO) == null) {
- Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
-
- Map<String, String> componentHosts = new HashMap<>();
+ if (componentHosts == null) {
+ componentHosts = new HashMap<>();
+ }
+ if (componentHosts.isEmpty()) {
+ Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster);
+ if (clusterHostInfo != null) {
- clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo);
+ clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo);
- for (Map.Entry<String, Set<String>> entry : clusterHostInfo.entrySet()) {
- componentHosts.put(entry.getKey(), StringUtils.join(entry.getValue(), ","));
+ for (Map.Entry<String, Set<String>> entry : clusterHostInfo.entrySet()) {
+ componentHosts.put(entry.getKey(), StringUtils.join(entry.getValue(), ","));
+ }
+ }
+ }
+ if (!componentHosts.isEmpty()) {
+ configurations.put(CLUSTER_HOST_INFO, componentHosts);
}
-
- configurations.put(CLUSTER_HOST_INFO, componentHosts);
}
- configurations.put("principals", principalNames(cluster, configurations));
+ configurations.put("principals", principalNames(cluster, configurations, userDescriptor, desiredConfigs));
return configurations;
}
+ /**
+ * Add configurations related to Kerberos, to a previously created map of configurations.
+ * <p/>
+ * The supplied map of configurations is expected to be mutable and will be altered.
+ *
+ * @param cluster the cluster
+ * @param configurations a map of configurations
+ * @param hostname a hostname
+ * @param kerberosDescriptorProperties the Kerberos descriptor properties
+ * @return the supplied map of configurations with updates applied
+ * @throws AmbariException
+ */
+ private Map<String, Map<String, String>> addAdditionalConfigurations(Cluster cluster, Map<String, Map<String, String>> configurations,
+ String hostname, Map<String, String> kerberosDescriptorProperties,
+ KerberosDescriptor userDescriptor)
+ throws AmbariException {
+
+ return addAdditionalConfigurations(cluster, configurations, hostname, kerberosDescriptorProperties,
+ userDescriptor, null, null);
+ }
+
/**
* Creates a deep copy of a map of maps, typically used to copy configuration sets.
*
@@ -3048,7 +3125,7 @@ public class KerberosHelperImpl implements KerberosHelper {
* @param cluster the cluster
* @return a Kerberos descriptor
*/
- private KerberosDescriptor getKerberosDescriptorUpdates(Cluster cluster) throws AmbariException {
+ public KerberosDescriptor getKerberosDescriptorUpdates(Cluster cluster) {
// find instance using name and foreign keys
TreeMap<String, String> foreignKeys = new TreeMap<>();
foreignKeys.put("cluster", String.valueOf(cluster.getClusterId()));
@@ -3188,7 +3265,7 @@ public class KerberosHelperImpl implements KerberosHelper {
*/
private Map<String, Map<String, String>> addConfigurationsForPreProcessedServices(Map<String, Map<String, String>> configurations,
Cluster cluster,
- KerberosDescriptor kerberosDescriptor,
+ @Nullable KerberosDescriptor kerberosDescriptor,
boolean calculateClusterHostInfo)
throws AmbariException {
@@ -3839,7 +3916,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Stage stage = createNewStage(id, cluster, requestId, requestContext, commandParams, hostParams);
stage.addServerActionCommand(actionClass.getName(), null, Role.AMBARI_SERVER_ACTION,
RoleCommand.EXECUTE, cluster.getClusterName(), event, commandParameters, commandDetail,
- ambariManagementController.findConfigurationTagsWithOverrides(cluster, null), timeout,
+ ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null), timeout,
retryAllowed, false);
return stage;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProvider.java
index 9dad423557..2677c5c919 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProvider.java
@@ -159,7 +159,10 @@ public class ClusterKerberosDescriptorResourceProvider extends ReadOnlyResourceP
cluster,
getEvaluateWhen(requestInfoProperties),
getAdditionalServices(requestInfoProperties),
- false);
+ false,
+ null,
+ null
+ );
} catch (AmbariException e) {
throw new SystemException("An unexpected error occurred building the cluster's composite Kerberos Descriptor", e);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
index b4f6def212..c16eb11cef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProvider.java
@@ -169,7 +169,7 @@ public class HostKerberosIdentityResourceProvider extends ReadOnlyResourceProvid
// Retrieve the active identities for the cluster filtered and grouped by hostname
Map<String, Collection<KerberosIdentityDescriptor>> hostDescriptors =
kerberosHelper.getActiveIdentities(clusterName, hostName, null, null, true, null,
- kerberosDescriptor);
+ kerberosDescriptor, null);
if (hostDescriptors != null) {
for (Map.Entry<String, Collection<KerberosIdentityDescriptor>> entry : hostDescriptors.entrySet()) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java
index 896abbe51c..64b1dcbfa3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProvider.java
@@ -332,7 +332,7 @@ public class RestMetricsPropertyProvider extends ThreadPoolEnabledPropertyProvid
if (configType != null && propertyName != null) {
try {
Map<String, Map<String, String>> configTags =
- amc.findConfigurationTagsWithOverrides(cluster, hostname);
+ amc.findConfigurationTagsWithOverrides(cluster, hostname, null);
if (configTags.containsKey(configType)) {
Map<String, Map<String, String>> properties = amc.getConfigHelper().getEffectiveConfigProperties(cluster,
Collections.singletonMap(configType, configTags.get(configType)));
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
index 231f92cf54..0d82caad62 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/AgentCommandsPublisher.java
@@ -35,8 +35,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.ExecutionException;
+
+import javax.annotation.Nullable;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.agent.AgentCommand;
import org.apache.ambari.server.agent.CancelCommand;
import org.apache.ambari.server.agent.ExecutionCommand;
@@ -50,8 +54,10 @@ import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabC
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.ambari.server.utils.ThreadPools;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections.CollectionUtils;
@@ -83,40 +89,72 @@ public class AgentCommandsPublisher {
@Inject
private AgentConfigsHolder agentConfigsHolder;
- public void sendAgentCommand(Multimap<Long, AgentCommand> agentCommands) throws AmbariException {
+ @Inject
+ private ThreadPools threadPools;
+
+
+ public void sendAgentCommand(Multimap<Long, AgentCommand> agentCommands) throws AmbariRuntimeException {
if (agentCommands != null && !agentCommands.isEmpty()) {
Map<Long, TreeMap<String, ExecutionCommandsCluster>> executionCommandsClusters = new TreeMap<>();
- for (Map.Entry<Long, AgentCommand> acHostEntry : agentCommands.entries()) {
- Long hostId = acHostEntry.getKey();
- AgentCommand ac = acHostEntry.getValue();
- populateExecutionCommandsClusters(executionCommandsClusters, hostId, ac);
- }
- for (Map.Entry<Long, TreeMap<String, ExecutionCommandsCluster>> hostEntry : executionCommandsClusters.entrySet()) {
- Long hostId = hostEntry.getKey();
- ExecutionCommandEvent executionCommandEvent = new ExecutionCommandEvent(hostId,
- agentConfigsHolder
- .initializeDataIfNeeded(hostId, true).getTimestamp(),
- hostEntry.getValue());
- STOMPUpdatePublisher.publish(executionCommandEvent);
- }
+ Map<Long, Map<String, DesiredConfig>> clusterDesiredConfigs = new HashMap<>();
+
+ try {
+ threadPools.getAgentPublisherCommandsPool().submit(() -> {
+ agentCommands.entries().stream().parallel().forEach(acHostEntry -> {
+ Long hostId = acHostEntry.getKey();
+ AgentCommand ac = acHostEntry.getValue();
+ Long clusterId = null;
+
+ if (ac instanceof ExecutionCommand) {
+ try {
+ clusterId = Long.valueOf(((ExecutionCommand)ac).getClusterId());
+ if (!clusterDesiredConfigs.containsKey(clusterId)) {
+ clusterDesiredConfigs.put(clusterId, clusters.getCluster(clusterId).getDesiredConfigs());
+ }
+ } catch (NumberFormatException|AmbariException ignored) {}
+ }
+
+ Map<String, DesiredConfig> desiredConfigs = (clusterId != null && clusterDesiredConfigs.containsKey(clusterId))
+ ? clusterDesiredConfigs.get(clusterId)
+ : null;
+ populateExecutionCommandsClusters(executionCommandsClusters, hostId, ac, desiredConfigs);
+ });
+ }).get();
+ } catch (InterruptedException|ExecutionException ignored) {}
+
+ try {
+ threadPools.getAgentPublisherCommandsPool().submit(() -> {
+ executionCommandsClusters.entrySet().stream().parallel().forEach(entry -> {
+ STOMPUpdatePublisher.publish(new ExecutionCommandEvent(
+ entry.getKey(),
+ agentConfigsHolder.initializeDataIfNeeded(entry.getKey(), true).getTimestamp(),
+ entry.getValue()
+ ));
+ });
+ }).get();
+ } catch (InterruptedException|ExecutionException ignored) {}
}
}
- public void sendAgentCommand(Long hostId, AgentCommand agentCommand) throws AmbariException {
+ public void sendAgentCommand(Long hostId, AgentCommand agentCommand) throws AmbariRuntimeException {
Multimap<Long, AgentCommand> agentCommands = ArrayListMultimap.create();
agentCommands.put(hostId, agentCommand);
sendAgentCommand(agentCommands);
}
- private void populateExecutionCommandsClusters(Map<Long, TreeMap<String, ExecutionCommandsCluster>> executionCommandsClusters,
- Long hostId, AgentCommand ac) throws AmbariException {
- try {
- if (LOG.isDebugEnabled()) {
+ private void populateExecutionCommandsClusters(
+ Map<Long, TreeMap<String, ExecutionCommandsCluster>> executionCommandsClusters,
+ Long hostId, AgentCommand ac,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariRuntimeException {
+
+ if (LOG.isDebugEnabled()) {
+ try {
LOG.debug("Sending command string = " + StageUtils.jaxbToString(ac));
+ } catch (Exception e) {
+ throw new AmbariRuntimeException("Could not get jaxb string for command", e);
}
- } catch (Exception e) {
- throw new AmbariException("Could not get jaxb string for command", e);
}
+
switch (ac.getCommandType()) {
case BACKGROUND_EXECUTION_COMMAND:
case EXECUTION_COMMAND: {
@@ -129,16 +167,20 @@ public class AgentCommandsPublisher {
if (SET_KEYTAB.equalsIgnoreCase(customCommand) || REMOVE_KEYTAB.equalsIgnoreCase(customCommand) || CHECK_KEYTABS.equalsIgnoreCase(customCommand)) {
LOG.info(String.format("%s called", customCommand));
try {
- injectKeytab(ec, customCommand, clusters.getHostById(hostId).getHostName());
+ injectKeytab(ec, customCommand, clusters.getHostById(hostId).getHostName(), desiredConfigs);
} catch (IOException e) {
- throw new AmbariException("Could not inject keytab into command", e);
+ throw new AmbariRuntimeException("Could not inject keytab into command", e);
}
}
}
String clusterName = ec.getClusterName();
String clusterId = "-1";
if (clusterName != null) {
- clusterId = Long.toString(clusters.getCluster(clusterName).getClusterId());
+ try {
+ clusterId = Long.toString(clusters.getCluster(clusterName).getClusterId());
+ } catch (AmbariException e) {
+ throw new AmbariRuntimeException(e);
+ }
}
ec.setClusterId(clusterId);
prepareExecutionCommandsClusters(executionCommandsClusters, hostId, clusterId);
@@ -176,12 +218,14 @@ public class AgentCommandsPublisher {
* @param ec the ExecutionCommand to update
* @param command a name of the relevant keytab command
* @param targetHost a name of the host the relevant command is destined for
+ * @param desiredConfigs desired config map
* @throws AmbariException
*/
- private void injectKeytab(ExecutionCommand ec, String command, String targetHost) throws AmbariException {
+ private void injectKeytab(ExecutionCommand ec, String command, String targetHost,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
KerberosCommandParameterProcessor processor = KerberosCommandParameterProcessor.getInstance(command, clusters, ec, kerberosKeytabController);
if (processor != null) {
- ec.setKerberosCommandParams(processor.process(targetHost));
+ ec.setKerberosCommandParams(processor.process(targetHost, desiredConfigs));
}
}
@@ -238,28 +282,31 @@ public class AgentCommandsPublisher {
* Kerberos-specific command details to send to the agent.
*
* @param targetHost the hostname of the target host
+ * @param desiredConfigs desired config map
* @return a map of propoperties to set as the Kerberos command parameters
* @throws AmbariException
*/
- public List<Map<String, String>> process(String targetHost) throws AmbariException {
+ public List<Map<String, String>> process(String targetHost,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
KerberosServerAction.KerberosCommandParameters kerberosCommandParameters = new KerberosServerAction.KerberosCommandParameters(executionCommand);
-
- try {
- Map<String, ? extends Collection<String>> serviceComponentFilter = getServiceComponentFilter(kerberosCommandParameters.getServiceComponentFilter());
- final Collection<KerberosIdentityDescriptor> serviceIdentities = serviceComponentFilter == null ? null : kerberosKeytabController.getServiceIdentities(executionCommand.getClusterName(), serviceComponentFilter.keySet());
- final Set<ResolvedKerberosKeytab> keytabsToInject = kerberosKeytabController.getFilteredKeytabs(serviceIdentities, kerberosCommandParameters.getHostFilter(), kerberosCommandParameters.getIdentityFilter());
- for (ResolvedKerberosKeytab resolvedKeytab : keytabsToInject) {
- for (ResolvedKerberosPrincipal resolvedPrincipal : resolvedKeytab.getPrincipals()) {
- String hostName = resolvedPrincipal.getHostName();
-
- if (targetHost.equalsIgnoreCase(hostName)) {
+ Map<String, ? extends Collection<String>> serviceComponentFilter = getServiceComponentFilter(kerberosCommandParameters.getServiceComponentFilter());
+ final Collection<KerberosIdentityDescriptor> serviceIdentities = serviceComponentFilter == null
+ ? null
+ : kerberosKeytabController.getServiceIdentities(executionCommand.getClusterName(), serviceComponentFilter.keySet(), desiredConfigs);
+ final Set<ResolvedKerberosKeytab> keytabsToInject = kerberosKeytabController.getFilteredKeytabs(serviceIdentities, kerberosCommandParameters.getHostFilter(), kerberosCommandParameters.getIdentityFilter());
+
+ keytabsToInject.forEach(resolvedKeytab -> resolvedKeytab.getPrincipals().forEach(
+ resolvedPrincipal -> {
+ String hostName = resolvedPrincipal.getHostName();
+ if (targetHost.equalsIgnoreCase(hostName)) {
+ try {
process(targetHost, resolvedKeytab, resolvedPrincipal, serviceComponentFilter);
+ } catch (IOException e) {
+ throw new AmbariRuntimeException("Could not inject keytabs to enable kerberos", e);
}
}
}
- } catch (IOException e) {
- throw new AmbariException("Could not inject keytabs to enable kerberos");
- }
+ ));
return kcp;
}
@@ -277,7 +324,9 @@ public class AgentCommandsPublisher {
* should be processed
* @throws IOException
*/
- protected void process(String hostName, ResolvedKerberosKeytab resolvedKeytab, ResolvedKerberosPrincipal resolvedPrincipal, Map<String, ? extends Collection<String>> serviceComponentFilter) throws IOException {
+ protected void process(String hostName, ResolvedKerberosKeytab resolvedKeytab,
+ ResolvedKerberosPrincipal resolvedPrincipal,
+ Map<String, ? extends Collection<String>> serviceComponentFilter) throws IOException {
Map<String, String> keytabMap = new HashMap<>();
keytabMap.put(KerberosIdentityDataFileReader.HOSTNAME, hostName);
keytabMap.put(KerberosIdentityDataFileReader.PRINCIPAL, resolvedPrincipal.getPrincipal());
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/STOMPUpdatePublisher.java b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/STOMPUpdatePublisher.java
index 0a1fb4d074..206d1863a5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/STOMPUpdatePublisher.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/publishers/STOMPUpdatePublisher.java
@@ -19,42 +19,34 @@ package org.apache.ambari.server.events.publishers;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.events.DefaultMessageEmitter;
import org.apache.ambari.server.events.STOMPEvent;
+import org.apache.ambari.server.utils.ThreadPools;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Singleton;
@Singleton
+@SuppressWarnings({"UnstableApiUsage", "rawtypes"})
public class STOMPUpdatePublisher {
private static final Logger LOG = LoggerFactory.getLogger(STOMPUpdatePublisher.class);
private final EventBus agentEventBus;
private final EventBus apiEventBus;
- private final ExecutorService threadPoolExecutorAgent = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("stomp-agent-bus-%d").build());
- private final ExecutorService threadPoolExecutorAPI = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("stomp-api-bus-%d").build());
+ private final List<BufferedUpdateEventPublisher> publishers = new ArrayList<>();
- public STOMPUpdatePublisher() throws NoSuchFieldException, IllegalAccessException {
- agentEventBus = new AsyncEventBus("agent-update-bus",
- threadPoolExecutorAgent);
- apiEventBus = new AsyncEventBus("api-update-bus",
- threadPoolExecutorAPI);
+ public STOMPUpdatePublisher() {
+ agentEventBus = new AsyncEventBus("agent-update-bus", ThreadPools.getSingleThreadedExecutor("stomp-agent-bus"));
+ apiEventBus = new AsyncEventBus("api-update-bus", ThreadPools.getSingleThreadedExecutor("stomp-api-bus"));
}
- private List<BufferedUpdateEventPublisher> publishers = new ArrayList<>();
-
public void registerPublisher(BufferedUpdateEventPublisher publisher) {
if (publishers.contains(publisher)) {
LOG.error("Publisher for type {} is already in use", publisher.getType());
@@ -74,6 +66,7 @@ public class STOMPUpdatePublisher {
}
}
+ @SuppressWarnings("unchecked")
private void publishAPI(STOMPEvent event) {
boolean published = false;
for (BufferedUpdateEventPublisher publisher : publishers) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
index c999f9b711..5b7927062e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabDAO.java
@@ -29,6 +29,7 @@ import javax.persistence.TypedQuery;
import org.apache.ambari.server.orm.RequiresSession;
import org.apache.ambari.server.orm.entities.KerberosKeytabEntity;
import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -88,6 +89,16 @@ public class KerberosKeytabDAO {
return entityManagerProvider.get().find(KerberosKeytabEntity.class, keytabPath);
}
+ @RequiresSession
+ public KerberosKeytabEntity findOrCreate(ResolvedKerberosKeytab resolvedKerberosKeytab) {
+ KerberosKeytabEntity result = find(resolvedKerberosKeytab.getFile());
+ if (result == null) {
+ result = new KerberosKeytabEntity(resolvedKerberosKeytab);
+ create(result);
+ }
+ return result;
+ }
+
@RequiresSession
public List<KerberosKeytabEntity> findByPrincipalAndHost(String principalName, Long hostId) {
Stopwatch stopwatch = Stopwatch.createStarted();
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
index eb43439865..6c2a48c64e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/KerberosKeytabPrincipalDAO.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import javax.annotation.Nullable;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
@@ -53,6 +54,9 @@ public class KerberosKeytabPrincipalDAO {
@Inject
HostDAO hostDAO;
+ @Inject
+ DaoUtils daoUtils;
+
@Transactional
public void create(KerberosKeytabPrincipalEntity kerberosKeytabPrincipalEntity) {
entityManagerProvider.get().persist(kerberosKeytabPrincipalEntity);
@@ -69,34 +73,49 @@ public class KerberosKeytabPrincipalDAO {
}
/**
- * Find or create {@link KerberosKeytabPrincipalEntity} with specified dependecies.
+ * Find or create {@link KerberosKeytabPrincipalEntity} with specified dependencies.
*
* @param kerberosKeytabEntity {@link KerberosKeytabEntity} which owns this principal
* @param hostEntity {@link HostEntity} which owns this principal
* @param kerberosPrincipalEntity {@link KerberosPrincipalEntity} which related to this principal
* @return evaluated entity
*/
- public KeytabPrincipalFindOrCreateResult findOrCreate(KerberosKeytabEntity kerberosKeytabEntity, HostEntity hostEntity, KerberosPrincipalEntity kerberosPrincipalEntity) {
+ public KeytabPrincipalFindOrCreateResult findOrCreate(KerberosKeytabEntity kerberosKeytabEntity,
+ HostEntity hostEntity, KerberosPrincipalEntity kerberosPrincipalEntity,
+ @Nullable List<KerberosKeytabPrincipalEntity> keytabList) {
KeytabPrincipalFindOrCreateResult result = new KeytabPrincipalFindOrCreateResult();
result.created = false;
Long hostId = hostEntity == null ? null : hostEntity.getHostId();
- KerberosKeytabPrincipalEntity kkp = findByNaturalKey(hostId, kerberosKeytabEntity.getKeytabPath(), kerberosPrincipalEntity.getPrincipalName());
- if (kkp == null) {
- result.created = true;
-
- kkp = new KerberosKeytabPrincipalEntity(
- kerberosKeytabEntity,
- hostEntity,
- kerberosPrincipalEntity
- );
- create(kkp);
-
- kerberosKeytabEntity.addKerberosKeytabPrincipal(kkp);
- kerberosPrincipalEntity.addKerberosKeytabPrincipal(kkp);
+ // The DB requests should be avoided due to heavy impact on the performance
+ KerberosKeytabPrincipalEntity kkp = (keytabList == null)
+ ? findByNaturalKey(hostId, kerberosKeytabEntity.getKeytabPath(), kerberosPrincipalEntity.getPrincipalName())
+ : keytabList.stream()
+ .filter(keytab ->
+ keytab.getHostId().equals(hostId)
+ && keytab.getKeytabPath().equals(kerberosKeytabEntity.getKeytabPath())
+ && keytab.getPrincipalName().equals(kerberosPrincipalEntity.getPrincipalName())
+ )
+ .findFirst()
+ .orElse(null);
+
+ if (kkp != null) {
+ result.kkp = kkp;
+ return result;
}
+ kkp = new KerberosKeytabPrincipalEntity(
+ kerberosKeytabEntity,
+ hostEntity,
+ kerberosPrincipalEntity
+ );
+ create(kkp);
+
+ kerberosKeytabEntity.addKerberosKeytabPrincipal(kkp);
+ kerberosPrincipalEntity.addKerberosKeytabPrincipal(kkp);
+
result.kkp = kkp;
+ result.created = true;
return result;
}
@@ -121,11 +140,7 @@ public class KerberosKeytabPrincipalDAO {
TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
createNamedQuery("KerberosKeytabPrincipalEntity.findByPrincipal", KerberosKeytabPrincipalEntity.class);
query.setParameter("principalName", principal);
- List<KerberosKeytabPrincipalEntity> result = query.getResultList();
- if (result == null) {
- return Collections.emptyList();
- }
- return result;
+ return daoUtils.selectList(query);
}
@RequiresSession
@@ -133,11 +148,7 @@ public class KerberosKeytabPrincipalDAO {
TypedQuery<KerberosKeytabPrincipalEntity> query = entityManagerProvider.get().
createNamedQuery("KerberosKeytabPrincipalEntity.findByHost", KerberosKeytabPrincipalEntity.class);
query.setParameter("hostId", hostId);
- List<KerberosKeytabPrincipalEntity> result = query.getResultList();
- if (result == null) {
- return Collections.emptyList();
- }
- return result;
+ return daoUtils.selectList(query);
}
@RequiresSession
@@ -146,11 +157,7 @@ public class KerberosKeytabPrincipalDAO {
createNamedQuery("KerberosKeytabPrincipalEntity.findByHostAndKeytab", KerberosKeytabPrincipalEntity.class);
query.setParameter("hostId", hostId);
query.setParameter("keytabPath", keytabPath);
- List<KerberosKeytabPrincipalEntity> result = query.getResultList();
- if (result == null) {
- return Collections.emptyList();
- }
- return result;
+ return daoUtils.selectList(query);
}
@RequiresSession
@@ -160,12 +167,7 @@ public class KerberosKeytabPrincipalDAO {
query.setParameter("hostId", hostId);
query.setParameter("keytabPath", keytabPath);
query.setParameter("principalName", principalName);
- List<KerberosKeytabPrincipalEntity> result = query.getResultList();
- if (result == null || result.size() == 0) {
- return null;
- } else {
- return result.get(0);
- }
+ return daoUtils.selectOne(query);
}
@RequiresSession
@@ -174,12 +176,7 @@ public class KerberosKeytabPrincipalDAO {
createNamedQuery("KerberosKeytabPrincipalEntity.findByKeytabAndPrincipalNullHost", KerberosKeytabPrincipalEntity.class);
query.setParameter("keytabPath", keytabPath);
query.setParameter("principalName", principal);
- List<KerberosKeytabPrincipalEntity> result = query.getResultList();
- if (result == null || result.size() == 0) {
- return null;
- } else {
- return result.get(0);
- }
+ return daoUtils.selectOne(query);
}
/**
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ArtifactEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ArtifactEntity.java
index 542017889e..9314a61983 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ArtifactEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ArtifactEntity.java
@@ -28,9 +28,13 @@ import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.NamedQueries;
import javax.persistence.NamedQuery;
+import javax.persistence.QueryHint;
import javax.persistence.Table;
import javax.persistence.Transient;
+import org.eclipse.persistence.config.HintValues;
+import org.eclipse.persistence.config.QueryHints;
+
import com.google.gson.Gson;
/**
* Entity representing an Artifact.
@@ -38,15 +42,22 @@ import com.google.gson.Gson;
@IdClass(ArtifactEntityPK.class)
@Table(name = "artifact")
@NamedQueries({
- @NamedQuery(name = "artifactByNameAndForeignKeys",
- query = "SELECT artifact FROM ArtifactEntity artifact " +
- "WHERE artifact.artifactName=:artifactName AND artifact.foreignKeys=:foreignKeys"),
- @NamedQuery(name = "artifactByName",
- query = "SELECT artifact FROM ArtifactEntity artifact " +
- "WHERE artifact.artifactName=:artifactName"),
- @NamedQuery(name = "artifactByForeignKeys",
- query = "SELECT artifact FROM ArtifactEntity artifact " +
- "WHERE artifact.foreignKeys=:foreignKeys")
+ @NamedQuery(
+ name = "artifactByNameAndForeignKeys",
+ query = "SELECT artifact FROM ArtifactEntity artifact WHERE artifact.artifactName=:artifactName AND artifact.foreignKeys=:foreignKeys",
+ hints = {
+ @QueryHint(name = QueryHints.QUERY_RESULTS_CACHE, value = HintValues.TRUE),
+ @QueryHint(name = QueryHints.QUERY_RESULTS_CACHE_SIZE, value = "100")
+ }
+ ),
+ @NamedQuery(
+ name = "artifactByName",
+ query = "SELECT artifact FROM ArtifactEntity artifact WHERE artifact.artifactName=:artifactName"
+ ),
+ @NamedQuery(
+ name = "artifactByForeignKeys",
+ query = "SELECT artifact FROM ArtifactEntity artifact WHERE artifact.foreignKeys=:foreignKeys"
+ )
})
@Entity
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
index d9a60ba038..b2caa300dd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/KerberosKeytabEntity.java
@@ -32,6 +32,8 @@ import javax.persistence.OneToMany;
import javax.persistence.QueryHint;
import javax.persistence.Table;
+import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
+
@Entity
@Table(name = "kerberos_keytab")
@NamedQueries({
@@ -77,6 +79,16 @@ public class KerberosKeytabEntity {
setKeytabPath(keytabPath);
}
+ public KerberosKeytabEntity(ResolvedKerberosKeytab resolvedKerberosKeytab) {
+ setKeytabPath(resolvedKerberosKeytab.getFile());
+ setAmbariServerKeytab(resolvedKerberosKeytab.isAmbariServerKeytab());
+ setWriteAmbariJaasFile(resolvedKerberosKeytab.isMustWriteAmbariJaasFile());
+ setOwnerName(resolvedKerberosKeytab.getOwnerName());
+ setOwnerAccess(resolvedKerberosKeytab.getOwnerAccess());
+ setGroupName(resolvedKerberosKeytab.getGroupName());
+ setGroupAccess(resolvedKerberosKeytab.getGroupAccess());
+ }
+
public String getKeytabPath() {
return keytabPath;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceStackEntity.java
similarity index 61%
copy from ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java
copy to ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceStackEntity.java
index b26f10634e..b134e031d3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/AmbariRuntimeException.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceStackEntity.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,17 +16,23 @@
* limitations under the License.
*/
-package org.apache.ambari.server;
+package org.apache.ambari.server.orm.entities;
-/**
- * Ambari unchecked exception.
- */
-public class AmbariRuntimeException extends RuntimeException {
- public AmbariRuntimeException(String message, Throwable cause) {
- super(message, cause);
+public class ServiceStackEntity {
+ private final String serviceName;
+ private final StackEntity stackEntity;
+
+
+ public ServiceStackEntity(String serviceName, StackEntity stackEntity) {
+ this.serviceName = serviceName;
+ this.stackEntity = stackEntity;
+ }
+
+ public String getServiceName() {
+ return serviceName;
}
- public AmbariRuntimeException(String message) {
- super(message);
+ public StackEntity getStackEntity() {
+ return stackEntity;
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java
index a2a5698488..c964b1562d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java
@@ -29,6 +29,9 @@ import javax.persistence.Table;
import javax.persistence.TableGenerator;
import javax.persistence.UniqueConstraint;
+import org.eclipse.persistence.config.HintValues;
+import org.eclipse.persistence.config.QueryHints;
+
/**
* The {@link StackEntity} class is used to model an alert that needs
* to run in the system. Each received alert from an agent will essentially be
@@ -39,13 +42,19 @@ import javax.persistence.UniqueConstraint;
"stack_name", "stack_version" }))
@TableGenerator(name = "stack_id_generator", table = "ambari_sequences", pkColumnName = "sequence_name", valueColumnName = "sequence_value", pkColumnValue = "stack_id_seq", initialValue = 0)
@NamedQueries({
- @NamedQuery(name = "StackEntity.findAll", query = "SELECT stack FROM StackEntity stack"),
@NamedQuery(name = "StackEntity.findByMpack", query = "SELECT stack FROM StackEntity stack where stack.mpackId = :mpackId"),
- @NamedQuery(name = "StackEntity.findByNameAndVersion", query = "SELECT stack FROM StackEntity stack WHERE stack.stackName = :stackName AND stack.stackVersion = :stackVersion",
- hints = {
- @QueryHint(name = "eclipselink.query-results-cache", value = "true"),
- @QueryHint(name = "eclipselink.query-results-cache.size", value = "100")
- })
+ @NamedQuery(
+ name = "StackEntity.findAll",
+ query = "SELECT stack FROM StackEntity stack"
+ ),
+ @NamedQuery(
+ name = "StackEntity.findByNameAndVersion",
+ query = "SELECT stack FROM StackEntity stack WHERE stack.stackName = :stackName AND stack.stackVersion = :stackVersion",
+ hints = {
+ @QueryHint(name = QueryHints.QUERY_RESULTS_CACHE, value = HintValues.TRUE),
+ @QueryHint(name = QueryHints.QUERY_RESULTS_CACHE_SIZE, value = "100")
+ }
+ )
})
public class StackEntity {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
index 72a4e14ed2..240e95acca 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
@@ -35,6 +35,8 @@ import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.controller.UpdateConfigurationPolicy;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
@@ -71,6 +73,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
@Inject
private ConfigHelper configHelper;
+ @Inject
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
+
@Override
protected CommandReport processIdentity(ResolvedKerberosPrincipal resolvedPrincipal, KerberosOperationHandler operationHandler, Map<String, String> kerberosConfiguration, boolean includedInFilter, Map<String, Object> requestSharedDataContext) throws AmbariException {
throw new UnsupportedOperationException();
@@ -153,11 +158,7 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
if (!StringUtils.isEmpty(hostName)) {
// Update the configurations with the relevant hostname
- Map<String, String> generalProperties = currentConfigurations.get("");
- if (generalProperties == null) {
- generalProperties = new HashMap<>();
- currentConfigurations.put("", generalProperties);
- }
+ Map<String, String> generalProperties = currentConfigurations.computeIfAbsent("", k -> new HashMap<>());
// Add the current hostname under "host" and "hostname"
generalProperties.put("host", hostName);
@@ -215,10 +216,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
propertiesToBeIgnored.putAll(propertiesToIgnore);
}
- // create database records for keytabs that must be presented on cluster
- for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) {
- kerberosHelper.createResolvedKeytab(keytab);
- }
+ // create database records for keytab that must be presented on cluster
+ List<KerberosKeytabPrincipalEntity> keytabList = kerberosKeytabPrincipalDAO.findAll();
+ resolvedKeytabs.values().forEach(keytab -> kerberosHelper.createResolvedKeytab(keytab, keytabList));
} catch (IOException e) {
String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
LOG.error(message, e);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
index 1b4f2069cc..a72f57a1f2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerAction.java
@@ -228,7 +228,7 @@ public class ConfigureAmbariIdentitiesServerAction extends KerberosServerAction
for(Map.Entry<String, String> mapping : principal.getServiceMapping().entries()) {
String serviceName = mapping.getKey();
String componentName = mapping.getValue();
- KerberosKeytabPrincipalEntity entity = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, kpe).kkp;
+ KerberosKeytabPrincipalEntity entity = kerberosKeytabPrincipalDAO.findOrCreate(kke, hostEntity, kpe, null).kkp;
entity.setDistributed(true);
entity.putServiceMapping(serviceName, componentName);
kerberosKeytabPrincipalDAO.merge(entity);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
index 7d73dc81f2..f1d546151d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java
@@ -57,6 +57,7 @@ import com.google.inject.Inject;
* {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, boolean, Map)}
* is invoked attempting the creation of the relevant keytab file.
*/
+@SuppressWarnings("UnstableApiUsage")
public class CreateKeytabFilesServerAction extends KerberosServerAction {
private final static Logger LOG = LoggerFactory.getLogger(CreateKeytabFilesServerAction.class);
@@ -159,6 +160,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
String message = null;
Set<ResolvedKerberosKeytab> keytabsToCreate = kerberosKeytabController.getFromPrincipal(resolvedPrincipal);
+ KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
try {
String dataDirectory = getDataDirectoryPath();
@@ -180,7 +182,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
String hostName = resolvedPrincipal.getHostName();
String keytabFilePath = rkk.getFile();
- if ((hostName != null) && !hostName.isEmpty() && (keytabFilePath != null) && !keytabFilePath.isEmpty()) {
+ if (hostName != null && !hostName.isEmpty() && keytabFilePath != null && !keytabFilePath.isEmpty()) {
Lock lock = m_locksByKeytab.get(keytabFilePath);
lock.lock();
@@ -219,7 +221,6 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
regenerateKeytabs = false;
}
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(resolvedPrincipal.getPrincipal());
String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath();
if (password == null) {
@@ -246,7 +247,8 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
}
}
} else {
- Keytab keytab = createKeytab(resolvedPrincipal.getPrincipal(), password, keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog);
+ Keytab keytab = createKeytab(resolvedPrincipal.getPrincipal(), principalEntity, password,
+ keyNumber, operationHandler, visitedPrincipalKeys != null, true, actionLog);
if (keytab != null) {
try {
@@ -319,30 +321,23 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
* @return a Keytab
* @throws AmbariException
*/
- public Keytab createKeytab(String principal, String password, Integer keyNumber,
- KerberosOperationHandler operationHandler, boolean checkCache,
+ public Keytab createKeytab(String principal, KerberosPrincipalEntity principalEntity, String password,
+ Integer keyNumber, KerberosOperationHandler operationHandler, boolean checkCache,
boolean canCache, ActionLog actionLog) throws AmbariException {
+
LOG.debug("Creating keytab for {} with kvno {}", principal, keyNumber);
Keytab keytab = null;
// Possibly get the keytab from the cache
if (checkCache) {
// Attempt to pull the keytab from the cache...
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal);
String cachedKeytabPath = (principalEntity == null) ? null : principalEntity.getCachedKeytabPath();
if (cachedKeytabPath != null) {
try {
keytab = Keytab.read(new File(cachedKeytabPath));
} catch (IOException e) {
- String message = String.format("Failed to read the cached keytab for %s, recreating if possible - %s",
- principal, e.getMessage());
-
- if (LOG.isDebugEnabled()) {
- LOG.warn(message, e);
- } else {
- LOG.warn(message);
- }
+ LOG.warn("Failed to read the cached keytab for {}, recreating if possible - {}", principal, e.getMessage());
}
}
}
@@ -354,17 +349,16 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction {
// If the current identity does not represent a service, copy it to a secure location
// and store that location so it can be reused rather than recreate it.
- KerberosPrincipalEntity principalEntity = kerberosPrincipalDAO.find(principal);
if (principalEntity != null) {
if (canCache) {
File cachedKeytabFile = cacheKeytab(principal, keytab);
String previousCachedFilePath = principalEntity.getCachedKeytabPath();
- String cachedKeytabFilePath = (!cachedKeytabFile.exists())
- ? null
- : cachedKeytabFile.getAbsolutePath();
+ String cachedKeytabFilePath = (cachedKeytabFile.exists()) ? cachedKeytabFile.getAbsolutePath() : null;
- principalEntity.setCachedKeytabPath(cachedKeytabFilePath);
- kerberosPrincipalDAO.merge(principalEntity);
+ if (previousCachedFilePath != null && !previousCachedFilePath.equals(cachedKeytabFilePath)) {
+ principalEntity.setCachedKeytabPath(cachedKeytabFilePath);
+ kerberosPrincipalDAO.merge(principalEntity);
+ }
if (previousCachedFilePath != null) {
if (!new File(previousCachedFilePath).delete()) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
index 5af81cfccb..2336fdebb3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java
@@ -51,6 +51,7 @@ import com.google.inject.Inject;
* {@link KerberosServerAction#processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, boolean, Map)}
* is invoked attempting the creation of the relevant principal.
*/
+@SuppressWarnings("UnstableApiUsage")
public class CreatePrincipalsServerAction extends KerberosServerAction {
private final static Logger LOG = LoggerFactory.getLogger(CreatePrincipalsServerAction.class);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
index 12369c6057..39f0f3de43 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/FinalizeKerberosServerAction.java
@@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.util.concurrent.Striped;
import com.google.inject.Inject;
+@SuppressWarnings("UnstableApiUsage")
public class FinalizeKerberosServerAction extends KerberosServerAction {
private final static Logger LOG = LoggerFactory.getLogger(FinalizeKerberosServerAction.class);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
index 437261e412..0c275e3f0d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java
@@ -21,20 +21,16 @@ package org.apache.ambari.server.serveraction.kerberos;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
@@ -46,20 +42,19 @@ import org.apache.ambari.server.orm.entities.HostEntity;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.serveraction.AbstractServerAction;
import org.apache.ambari.server.serveraction.kerberos.stageutils.KerberosKeytabController;
-import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosPrincipal;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
import org.apache.ambari.server.utils.StageUtils;
+import org.apache.ambari.server.utils.ThreadPools;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.reflect.TypeToken;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.inject.Inject;
/**
@@ -202,6 +197,9 @@ public abstract class KerberosServerAction extends AbstractServerAction {
@Inject
private Configuration configuration;
+ @Inject
+ private ThreadPools threadPools;
+
/**
* Given a (command parameter) Map and a property name, attempts to safely retrieve the requested
* data.
@@ -416,7 +414,7 @@ public abstract class KerberosServerAction extends AbstractServerAction {
* Using {@link #getHostFilter()}, {@link #getIdentityFilter()} and {@link #getServiceComponentFilter()} it retrieve
* list of filtered keytabs and their principals and process each principal using
* {@link #processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, boolean, Map)}.
- * The configuration option {@link Configuration#getKerberosServerActionThreadpoolSize()} defines
+ * The configuration option {@link Configuration#getKerberosServerActionThreadPoolSize()} defines
* how many threads will handle {@link #processIdentity(ResolvedKerberosPrincipal, KerberosOperationHandler, Map, boolean, Map)}.
* The default is {@code 1}, but this method must be thread-safe in the event that concurrent threads are used.
*
@@ -427,7 +425,7 @@ public abstract class KerberosServerAction extends AbstractServerAction {
*/
protected CommandReport processIdentities(Map<String, Object> requestSharedDataContext)
throws AmbariException {
- CommandReport commandReport = null;
+ final CommandReport[] commandReport = {null};
Map<String, String> commandParameters = getCommandParameters();
actionLog.writeStdOut("Processing identities...");
@@ -455,66 +453,45 @@ public abstract class KerberosServerAction extends AbstractServerAction {
try {
// create the thread factory, executor, and completion service for
// running the identity processing in parallel
- String factoryName = "process-identity-%d";
- ExecutionCommand executionCommand = getExecutionCommand();
- if( null != executionCommand ) {
- factoryName = "process-identity-task-" + executionCommand.getTaskId() + "-thread-%d";
- }
- ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(factoryName).build();
+ ExecutionCommand executionCommand = getExecutionCommand();
+ int threadCount = configuration.getKerberosServerActionThreadPoolSize();
+ String factoryName = (executionCommand == null)
+ ? "process-identity-%d"
+ : "process-identity-task-" + executionCommand.getTaskId() + "-thread-%d";
- int threadCount = configuration.getKerberosServerActionThreadpoolSize();
- ExecutorService executorService = Executors.newFixedThreadPool(threadCount, threadFactory);
- CompletionService<CommandReport> completionService = new ExecutorCompletionService<>(executorService);
Map<String, Collection<String>> serviceComponentFilter = getServiceComponentFilter();
if (serviceComponentFilter != null && pruneServiceFilter()) {
- kerberosKeytabController.adjustServiceComponentFilter(clusters.getCluster(getClusterName()), true, serviceComponentFilter);
+ kerberosKeytabController.adjustServiceComponentFilter(clusters.getCluster(getClusterName()),
+ true, serviceComponentFilter);
}
- final Collection<KerberosIdentityDescriptor> serviceIdentities = serviceComponentFilter == null ? null : kerberosKeytabController.getServiceIdentities(getClusterName(), serviceComponentFilter.keySet());
- List<Future<CommandReport>> futures = new ArrayList<>();
- for (ResolvedKerberosKeytab rkk : kerberosKeytabController.getFilteredKeytabs(serviceIdentities, getHostFilter(),getIdentityFilter())) {
- for (ResolvedKerberosPrincipal principal : rkk.getPrincipals()) {
- // submit this method to the service to be processed concurrently
- Future<CommandReport> future = completionService.submit(() -> {
+ final Collection<KerberosIdentityDescriptor> serviceIdentities = (serviceComponentFilter == null)
+ ? null
+ : kerberosKeytabController.getServiceIdentities(getClusterName(), serviceComponentFilter.keySet(), null);
+
+ threadPools.parallelOperation(factoryName, threadCount, "identities",
+ kerberosKeytabController.getFilteredKeytabs(serviceIdentities, getHostFilter(), getIdentityFilter())
+ .stream().flatMap(rkk -> rkk.getPrincipals().stream()).map(principal -> (Callable<CommandReport>) () -> {
try {
return processIdentity(principal, handler, kerberosConfiguration,
- isRelevantIdentity(serviceIdentities, principal), requestSharedDataContext);
+ isRelevantIdentity(serviceIdentities, principal), requestSharedDataContext);
} catch (AmbariException ambariException) {
- throw new RuntimeException(ambariException);
+ throw new AmbariRuntimeException(ambariException);
}
- });
-
- // keep track of futures for total count and ability to cancel later
- futures.add(future);
- }
- }
-
- LOG.info("Processing {} identities concurrently...", futures.size());
-
- // get each future as it completes (out of order is OK), cancelling if
- // an error is found
- try {
- for( int i = 0; i < futures.size(); i++ ) {
- Future<CommandReport> future = completionService.take();
- commandReport = future.get();
-
- // If the principal processor returns a CommandReport, than it is time to stop
- // since an error condition has probably occurred, else all is assumed to be well.
- if (commandReport != null) {
- break;
}
+ ).collect(Collectors.toList()),
+ (cr) -> { // processIdentity returns null if operation completed ok, else FAILED report generated
+ boolean isFailed = Objects.isNull(cr);
+ if (!isFailed) {
+ commandReport[0] = cr;
+ }
+ return isFailed;
}
- } catch (Exception exception) {
- LOG.error("Unable to process identities asynchronously", exception);
- return createCommandReport(0, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(),actionLog.getStdErr());
- } finally {
- futures.stream()
- .filter(x -> !x.isCancelled() && !x.isDone())
- .forEach(x -> x.cancel(true));
-
- executorService.shutdown();
- }
+ );
+ } catch (Exception exception) {
+ LOG.error("Unable to process identities asynchronously", exception);
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
} finally {
// The KerberosOperationHandler needs to be closed, if it fails to close ignore the
// exception since there is little we can or care to do about it now.
@@ -532,9 +509,9 @@ public abstract class KerberosServerAction extends AbstractServerAction {
// If commandReport is null, we can assume this operation was a success, so return a successful
// CommandReport; else return the previously created CommandReport.
- return (commandReport == null)
+ return (commandReport[0] == null)
? createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr())
- : commandReport;
+ : commandReport[0];
}
protected boolean pruneServiceFilter() {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
index c3831006bd..0bf7b3d600 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/MITKerberosOperationHandler.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.security.credential.PrincipalKeyCredential;
import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
@@ -342,26 +341,22 @@ public class MITKerberosOperationHandler extends KDCKerberosOperationHandler {
@Override
protected String[] getKinitCommand(String executableKinit, PrincipalKeyCredential credentials, String credentialsCache, Map<String, String> kerberosConfiguration) throws KerberosOperationException {
// kinit -c <path> -S kadmin/`hostname -f` <principal>
- try {
- String kadminPrincipalName = variableReplacementHelper.replaceVariables(kerberosConfiguration.get(KERBEROS_ENV_KADMIN_PRINCIPAL_NAME), buildReplacementsMap(kerberosConfiguration));
- if (kadminPrincipalName == null) {
- kadminPrincipalName = String.format("kadmin/%s", getAdminServerHost(false));
- }
- String [] command = new String[]{
- executableKinit,
- "-c",
- credentialsCache,
- "-S",
- kadminPrincipalName,
- credentials.getPrincipal()
- };
- if (Arrays.asList(command).contains(null)){
- throw new KerberosOperationException("Got a null value, can not create 'kinit' command");
- }
- return command;
- } catch (AmbariException e) {
- throw new KerberosOperationException("Error while getting 'kinit' command", e);
+ String kadminPrincipalName = variableReplacementHelper.replaceVariables(kerberosConfiguration.get(KERBEROS_ENV_KADMIN_PRINCIPAL_NAME), buildReplacementsMap(kerberosConfiguration));
+ if (kadminPrincipalName == null) {
+ kadminPrincipalName = String.format("kadmin/%s", getAdminServerHost(false));
+ }
+ String [] command = new String[]{
+ executableKinit,
+ "-c",
+ credentialsCache,
+ "-S",
+ kadminPrincipalName,
+ credentials.getPrincipal()
+ };
+ if (Arrays.asList(command).contains(null)){
+ throw new KerberosOperationException("Got a null value, can not create 'kinit' command");
}
+ return command;
}
private Map<String, Map<String, String>> buildReplacementsMap(Map<String, String> kerberosConfiguration) {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
index 1ed9e40f2c..6e139c9024 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
@@ -104,7 +104,7 @@ public class PrepareDisableKerberosServerAction extends AbstractPrepareKerberosS
// Calculate the current host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false, null);
processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory,
configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
index a130df4637..50781448f9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
@@ -106,7 +106,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities
// Calculate the current host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false, null);
processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory,
configurations, kerberosConfigurations, true, propertiesToIgnore);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
index b6638faa5f..d8f3b52903 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
@@ -124,7 +124,7 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber
// Calculate the current host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false, null);
processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory,
configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore);
@@ -138,7 +138,7 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber
}
// TODO: Determine if we need to do this again since it is done a few lines above.
- Map<String, Map<String, String>> calculatedConfigurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> calculatedConfigurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false, null);
if (updateConfigurationPolicy.applyIdentityChanges()) {
processAuthToLocalRules(cluster, calculatedConfigurations, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters), false);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java
index bf3b20afad..08be6643ea 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerAction.java
@@ -139,17 +139,15 @@ public class UpdateKerberosConfigsServerAction extends AbstractServerAction {
String configNote = getCommandParameterValue(getCommandParameters(), KerberosServerAction.UPDATE_CONFIGURATION_NOTE);
- if((configNote == null) || configNote.isEmpty()) {
+ if(configNote == null || configNote.isEmpty()) {
configNote = cluster.getSecurityType() == SecurityType.KERBEROS
? "Enabling Kerberos"
: "Disabling Kerberos";
}
- for (String configType : configTypes) {
- configHelper.updateConfigType(cluster, cluster.getDesiredStackVersion(), controller,
- configType, propertiesToSet.get(configType), propertiesToRemove.get(configType),
- authenticatedUserName, configNote);
- }
+ configHelper.updateBulkConfigType(cluster, cluster.getDesiredStackVersion(), controller,
+ configTypes, propertiesToSet, propertiesToRemove,
+ authenticatedUserName, configNote);
} catch (IOException e) {
String message = "Could not update services configs to enable kerberos";
actionLog.writeStdErr(message);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
index cac811e1fa..3c1c2e6cd6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/stageutils/KerberosKeytabController.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
@@ -36,6 +38,7 @@ import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.orm.entities.KerberosPrincipalEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
@@ -290,37 +293,36 @@ public class KerberosKeytabController {
return adjustedFilter;
}
- public Collection<KerberosIdentityDescriptor> getServiceIdentities(String clusterName, Collection<String> services) throws AmbariException {
+ public Collection<KerberosIdentityDescriptor> getServiceIdentities(String clusterName, Collection<String> services,
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException {
+
final Collection<KerberosIdentityDescriptor> serviceIdentities = new ArrayList<>();
Cluster cluster = clusters.getCluster(clusterName);
+
+ if (desiredConfigs == null) {
+ desiredConfigs = cluster.getDesiredConfigs();
+ }
+
KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(cluster, false);
+ KerberosDescriptor userDescriptor = kerberosHelper.getKerberosDescriptorUpdates(cluster);
Map<String, Map<String, Map<String, String>>> hostConfigurations = new HashMap<>();
Map<String, Host> hostMap = clusters.getHostsForCluster(clusterName);
Set<String> hosts = new HashSet<>(hostMap.keySet());
+ Map<String, String> componentHosts = new HashMap<>();
- String ambariServerHostname = StageUtils.getHostName();
- if (!hosts.contains(ambariServerHostname)) {
- hosts.add(ambariServerHostname);
- }
- for( String hostName : hosts ) {
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(
- cluster, hostName, kerberosDescriptor, false, false);
- hostConfigurations.put(hostName, configurations);
+ hosts.add(StageUtils.getHostName());
+
+ for(String hostName: hosts) {
+ hostConfigurations.put(
+ hostName,
+ kerberosHelper.calculateConfigurations(cluster, hostName, kerberosDescriptor, userDescriptor,
+ false,false, componentHosts, desiredConfigs)
+ );
}
- for (String service : services) {
- Collection<Collection<KerberosIdentityDescriptor>> identities = kerberosHelper.getActiveIdentities(
- clusterName,
- null,
- service,
- null,
- true,
- hostConfigurations,
- kerberosDescriptor
- ).values();
-
- for (Collection<KerberosIdentityDescriptor> activeIdentities : identities) {
- serviceIdentities.addAll(activeIdentities);
- }
+ for (String service: services) {
+ kerberosHelper.getActiveIdentities(clusterName,null, service, null,true,
+ hostConfigurations,kerberosDescriptor, desiredConfigs).values().forEach(serviceIdentities::addAll);
}
return serviceIdentities;
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
index 8e207ae8d7..6cbb41cab4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
@@ -41,9 +41,9 @@ import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.controller.RootComponent;
import org.apache.ambari.server.controller.RootService;
import org.apache.ambari.server.orm.dao.HostDAO;
-import org.apache.ambari.server.orm.dao.KerberosKeytabDAO;
-import org.apache.ambari.server.orm.dao.KerberosPrincipalDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.orm.entities.HostEntity;
+import org.apache.ambari.server.orm.entities.KerberosKeytabPrincipalEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType;
import org.apache.ambari.server.serveraction.kerberos.stageutils.ResolvedKerberosKeytab;
@@ -94,10 +94,7 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
private HostDAO hostDAO;
@Inject
- private KerberosKeytabDAO kerberosKeytabDAO;
-
- @Inject
- KerberosPrincipalDAO kerberosPrincipalDAO;
+ private KerberosKeytabPrincipalDAO kerberosKeytabPrincipalDAO;
@Override
public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException {
@@ -125,11 +122,11 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
"The target stack Id was not specified.");
}
- KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(KerberosHelper.KerberosDescriptorType.COMPOSITE, cluster, stackId, true);
+ KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(KerberosHelper.KerberosDescriptorType.COMPOSITE, cluster, stackId, true, null);
// Calculate the current host-specific configurations. These will be used to replace
// variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, true, false);
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, true, false, null);
PreconfigureServiceType preconfigureServiceType = getPreconfigureServiceType(configurations);
@@ -388,10 +385,9 @@ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
propertiesToBeIgnored.putAll(propertiesToIgnore);
}
- // create database records for keytabs that must be presented on cluster
- for (ResolvedKerberosKeytab keytab : resolvedKeytabs.values()) {
- kerberosHelper.createResolvedKeytab(keytab);
- }
+ // create database records for keytab that must be presented on cluster
+ List<KerberosKeytabPrincipalEntity> keytabList = kerberosKeytabPrincipalDAO.findAll();
+ resolvedKeytabs.values().forEach(keytab -> kerberosHelper.createResolvedKeytab(keytab, keytabList));
} catch (IOException e) {
throw new AmbariException(e.getMessage(), e);
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
index 30fd197ab6..799e1dabb7 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ClusterResponse;
import org.apache.ambari.server.controller.ServiceConfigVersionResponse;
@@ -386,6 +388,15 @@ public interface Cluster {
*/
Config getDesiredConfigByType(String configType);
+ /**
+ * Gets the desired (and selected) config by type.
+ * @param configType the type of configuration
+ * @param desiredConfigs map of desired configs
+ * @return the {@link Config} instance, or <code>null</code> if the type has
+ * not been set.
+ */
+ Config getDesiredConfigByType(String configType, @Nullable Map<String, DesiredConfig> desiredConfigs);
+
/**
* Check if config type exists in cluster.
* @param configType the type of configuration
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index fbc8cacf08..5a58768a0d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -33,7 +33,10 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.agent.stomp.AgentConfigsHolder;
import org.apache.ambari.server.agent.stomp.MetadataHolder;
import org.apache.ambari.server.agent.stomp.dto.ClusterConfigs;
@@ -169,7 +172,7 @@ public class ConfigHelper {
* @throws AmbariException
*/
public Map<String, Map<String, String>> getEffectiveDesiredTags(
- Cluster cluster, String hostName) throws AmbariException {
+ Cluster cluster, String hostName) throws AmbariException {
return getEffectiveDesiredTags(cluster, hostName, null);
}
@@ -185,7 +188,8 @@ public class ConfigHelper {
* @throws AmbariException
*/
public Map<String, Map<String, String>> getEffectiveDesiredTags(Cluster cluster, String hostName,
- Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
+ @Nullable Map<String, DesiredConfig> desiredConfigs)
+ throws AmbariException {
Host host = (hostName == null) ? null : clusters.getHost(hostName);
Map<String, HostConfig> desiredHostConfigs = (host == null) ? null
@@ -210,9 +214,8 @@ public class ConfigHelper {
* penality.
* @return a map of tag type to tag names with overrides
*/
- private Map<String, Map<String, String>> getEffectiveDesiredTags(
- Cluster cluster, Map<String, DesiredConfig> clusterDesired,
- Map<String, HostConfig> hostConfigOverrides) {
+ private Map<String, Map<String, String>> getEffectiveDesiredTags(Cluster cluster, Map<String,
+ DesiredConfig> clusterDesired, Map<String, HostConfig> hostConfigOverrides) {
if (null == cluster) {
clusterDesired = new HashMap<>();
@@ -1118,6 +1121,52 @@ public class ConfigHelper {
return propertySets;
}
+ public void updateConfigType(Cluster cluster, StackId stackId, AmbariManagementController controller,
+ String configType, Map<String, String> updates,
+ Collection<String> removals,
+ String authenticatedUserName,
+ String serviceVersionNote) throws AmbariException {
+ updateConfigType(cluster, stackId, controller, configType, updates, removals, authenticatedUserName,
+ serviceVersionNote, null, true);
+ }
+
+ public void updateBulkConfigType(Cluster cluster, StackId stackId, AmbariManagementController controller,
+ Iterable<String> configTypes,
+ Map<String, Map<String, String>> updates,
+ Map<String, Collection<String>> removals,
+ String authenticatedUserName,
+ String serviceVersionNote) throws AmbariException, AmbariRuntimeException {
+
+ Map<String, DesiredConfig> desiredConfig = cluster.getDesiredConfigs();
+
+ Boolean[] doUpdateAgentConfigs = {false};
+ LOG.info("Bulk config update. Starting...");
+ configTypes.forEach(configType -> {
+ try {
+ Boolean updated = updateConfigType(
+ cluster, stackId, controller,
+ configType,
+ updates.get(configType),
+ removals.get(configType),
+ authenticatedUserName,
+ serviceVersionNote,
+ desiredConfig,
+ false
+ );
+ LOG.info("Bulk config update. Working with {}...{}", configType, updated ? "updated" : "not updated");
+ doUpdateAgentConfigs[0] = doUpdateAgentConfigs[0] || updated;
+ } catch (AmbariException e) {
+ throw new AmbariRuntimeException(e);
+ }
+ });
+
+ LOG.info("Bulk config update, agent update is {} required", (doUpdateAgentConfigs[0]) ? "" : "not");
+
+ if (doUpdateAgentConfigs[0]) {
+ updateAgentConfigs(Collections.singleton(cluster.getClusterName()));
+ }
+ }
+
/**
* A helper method to create a new {@link Config} for a given configuration
* type and updates to the current values, if any. This method will perform the following tasks:
@@ -1130,32 +1179,29 @@ public class ConfigHelper {
* <li>Create an entry in the configuration history with a note and username.</li>
* <ul>
*
- * @param cluster
- * @param controller
- * @param configType
- * @param updates
- * @param removals a collection of property names to remove from the configuration type
- * @param authenticatedUserName
- * @param serviceVersionNote
* @throws AmbariException
*/
- public void updateConfigType(Cluster cluster, StackId stackId,
- AmbariManagementController controller, String configType, Map<String, String> updates,
- Collection<String> removals, String authenticatedUserName, String serviceVersionNote)
- throws AmbariException {
+ public Boolean updateConfigType(Cluster cluster, StackId stackId, AmbariManagementController controller,
+ String configType, Map<String, String> updates,
+ Collection<String> removals,
+ String authenticatedUserName,
+ String serviceVersionNote,
+ @Nullable Map<String, DesiredConfig> desiredConfig,
+ Boolean doUpdateAgentConfigs) throws AmbariException {
// Nothing to update or remove
if (configType == null ||
(updates == null || updates.isEmpty()) &&
(removals == null || removals.isEmpty())) {
- return;
+ return false;
}
- Config oldConfig = cluster.getDesiredConfigByType(configType);
+ Config oldConfig = (desiredConfig != null && desiredConfig.containsKey(configType))
+ ? cluster.getConfig(configType, desiredConfig.get(configType).getTag())
+ : cluster.getDesiredConfigByType(configType);
Map<String, String> oldConfigProperties;
Map<String, String> properties = new HashMap<>();
- Map<String, Map<String, String>> propertiesAttributes =
- new HashMap<>();
+ Map<String, Map<String, String>> propertiesAttributes = new HashMap<>();
if (oldConfig == null) {
oldConfigProperties = null;
@@ -1183,14 +1229,17 @@ public class ConfigHelper {
}
}
- if ((oldConfigProperties == null)
- || !Maps.difference(oldConfigProperties, properties).areEqual()) {
- if (createConfigType(cluster, stackId, controller, configType, properties,
- propertiesAttributes, authenticatedUserName, serviceVersionNote)) {
-
+ if ((oldConfigProperties == null || !Maps.difference(oldConfigProperties, properties).areEqual())
+ && createConfigType(cluster, stackId, controller, configType, properties, propertiesAttributes,
+ authenticatedUserName, serviceVersionNote)) {
+ if (doUpdateAgentConfigs) {
updateAgentConfigs(Collections.singleton(cluster.getClusterName()));
}
+
+ return true;
}
+
+ return false;
}
public void createConfigType(Cluster cluster, StackId stackId,
@@ -2046,18 +2095,13 @@ public class ConfigHelper {
}
Map<String, Map<String, String>> configurations = new HashMap<>();
Map<String, Map<String, Map<String, String>>> configurationAttributes = new HashMap<>();
- if (LOG.isDebugEnabled()) {
- LOG.debug("For configs update on host {} will be used cluster entity {}", hostId, cl.getClusterEntity().toString());
- }
Map<String, DesiredConfig> clusterDesiredConfigs = cl.getDesiredConfigs(false);
- if (LOG.isDebugEnabled()) {
- LOG.debug("For configs update on host {} will be used following cluster desired configs {}", hostId,
- clusterDesiredConfigs.toString());
- }
+ Map<String, Map<String, String>> configTags = getEffectiveDesiredTags(cl, host.getHostName(), clusterDesiredConfigs);
- Map<String, Map<String, String>> configTags =
- getEffectiveDesiredTags(cl, host.getHostName(), clusterDesiredConfigs);
+ // Logging below creating too much spam and slowing down operations
if (LOG.isDebugEnabled()) {
+ LOG.debug("For configs update on host {} will be used cluster entity {}", hostId, cl.getClusterEntity().toString());
+ LOG.debug("For configs update on host {} will be used following cluster desired configs {}", hostId, clusterDesiredConfigs.toString());
LOG.debug("For configs update on host {} will be used following effective desired tags {}", hostId, configTags.toString());
}
@@ -2073,8 +2117,7 @@ public class ConfigHelper {
new ClusterConfigs(configurationsTreeMap, configurationAttributesTreeMap));
}
- AgentConfigsUpdateEvent agentConfigsUpdateEvent = new AgentConfigsUpdateEvent(hostId, clustersConfigs);
- return agentConfigsUpdateEvent;
+ return new AgentConfigsUpdateEvent(hostId, clustersConfigs);
}
private Map<String, Map<String, String>> unescapeConfigNames(Map<String, Map<String, String>> configurations) {
@@ -2094,21 +2137,13 @@ public class ConfigHelper {
Map<String, Map<String, Map<String, String>>> configurationAttributes) {
Map<String, Map<String, Map<String, String>>> unescapedConfigAttributes = new HashMap<>();
- for (Entry<String, Map<String, Map<String, String>>> configAttrTypeEntry : configurationAttributes.entrySet()) {
- unescapedConfigAttributes.put(configAttrTypeEntry.getKey(), unescapeConfigNames(configAttrTypeEntry.getValue()));
- }
-
+ configurationAttributes.forEach((key, value) -> unescapedConfigAttributes.put(key, unescapeConfigNames(value)));
return unescapedConfigAttributes;
}
public SortedMap<String, SortedMap<String, String>> sortConfigutations(Map<String, Map<String, String>> configurations) {
SortedMap<String, SortedMap<String, String>> configurationsTreeMap = new TreeMap<>();
- configurations.forEach((k, v) -> {
- TreeMap<String, String> c = new TreeMap<>();
- c.putAll(v);
- configurationsTreeMap.put(k, c);
- });
-
+ configurations.forEach((k, v) -> configurationsTreeMap.put(k, new TreeMap<>(v)));
return configurationsTreeMap;
}
@@ -2117,11 +2152,7 @@ public class ConfigHelper {
SortedMap<String, SortedMap<String, SortedMap<String, String>>> configurationAttributesTreeMap = new TreeMap<>();
configurationAttributes.forEach((k, v) -> {
SortedMap<String, SortedMap<String, String>> c = new TreeMap<>();
- v.forEach((k1, v1) -> {
- SortedMap<String, String> c1 = new TreeMap<>();
- c1.putAll(v1);
- c.put(k1, c1);
- });
+ v.forEach((k1, v1) -> c.put(k1, new TreeMap<>(v1)));
configurationAttributesTreeMap.put(k, c);
});
@@ -2141,7 +2172,12 @@ public class ConfigHelper {
public Map<String, Map<String, String>> calculateExistingConfigurations(AmbariManagementController ambariManagementController, Cluster cluster) throws AmbariException {
final Map<String, Map<String, String>> configurations = new HashMap<>();
for (Host host : cluster.getHosts()) {
- configurations.putAll(calculateExistingConfigurations(ambariManagementController, cluster, host.getHostName()));
+ configurations.putAll(calculateExistingConfigurations(
+ ambariManagementController,
+ cluster,
+ host.getHostName(),
+ null
+ ));
}
return configurations;
}
@@ -2152,15 +2188,20 @@ public class ConfigHelper {
* @param ambariManagementController the Ambari management controller
* @param cluster the cluster
* @param hostname a hostname
+ * @param desiredConfigs desired configuration map
* @return a map of the existing configurations
*/
- public Map<String, Map<String, String>> calculateExistingConfigurations(AmbariManagementController ambariManagementController, Cluster cluster, String hostname) throws AmbariException {
+ public Map<String, Map<String, String>> calculateExistingConfigurations(
+ AmbariManagementController ambariManagementController, Cluster cluster, String hostname,
+ @Nullable Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
+
// For a configuration type, both tag and an actual configuration can be stored
// Configurations from the tag is always expanded and then over-written by the actual
// global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==>
// global:{a1:A2,b1:B1,c1:C1}
final Map<String, Map<String, String>> configurations = new HashMap<>();
- final Map<String, Map<String, String>> configurationTags = ambariManagementController.findConfigurationTagsWithOverrides(cluster, hostname);
+ final Map<String, Map<String, String>> configurationTags = ambariManagementController
+ .findConfigurationTagsWithOverrides(cluster, hostname, desiredConfigs);
final Map<String, Map<String, String>> configProperties = getEffectiveConfigProperties(cluster, configurationTags);
// Apply the configurations saved with the Execution Cmd on top of
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
index cdf36f0f1d..f87bf427a9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
@@ -21,6 +21,8 @@ package org.apache.ambari.server.state;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.agent.AgentEnv;
import org.apache.ambari.server.agent.DiskInfo;
@@ -421,7 +423,7 @@ public interface Host extends Comparable {
* @throws AmbariException
*/
Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster,
- Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException;
+ @Nullable Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException;
/**
* Sets the maintenance state for the host.
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 8cafefd541..c9765086fe 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -956,7 +956,7 @@ public class ClusterImpl implements Cluster {
ClusterEntity clusterEntity = getClusterEntity();
clusterEntity.setDesiredStack(stackEntity);
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterDAO.merge(clusterEntity);
loadServiceConfigTypes();
} finally {
@@ -979,7 +979,7 @@ public class ClusterImpl implements Cluster {
@Override
public State getProvisioningState() {
- State provisioningState = null;
+ State provisioningState;
ClusterEntity clusterEntity = getClusterEntity();
provisioningState = clusterEntity.getProvisioningState();
@@ -994,7 +994,7 @@ public class ClusterImpl implements Cluster {
public void setProvisioningState(State provisioningState) {
ClusterEntity clusterEntity = getClusterEntity();
clusterEntity.setProvisioningState(provisioningState);
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterDAO.merge(clusterEntity);
}
private boolean setBlueprintProvisioningState(BlueprintProvisioningState blueprintProvisioningState) {
@@ -1033,7 +1033,7 @@ public class ClusterImpl implements Cluster {
public void setSecurityType(SecurityType securityType) {
ClusterEntity clusterEntity = getClusterEntity();
clusterEntity.setSecurityType(securityType);
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterDAO.merge(clusterEntity);
}
/**
@@ -1163,11 +1163,11 @@ public class ClusterImpl implements Cluster {
clusterStateDAO.create(clusterStateEntity);
clusterStateEntity = clusterStateDAO.merge(clusterStateEntity);
clusterEntity.setClusterStateEntity(clusterStateEntity);
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterDAO.merge(clusterEntity);
} else {
clusterStateEntity.setCurrentStack(stackEntity);
- clusterStateEntity = clusterStateDAO.merge(clusterStateEntity);
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterStateDAO.merge(clusterStateEntity);
+ clusterDAO.merge(clusterEntity);
}
} catch (RollbackException e) {
LOG.warn("Unable to set version " + stackId + " for cluster "
@@ -1207,6 +1207,14 @@ public class ClusterImpl implements Cluster {
}
}
+ @Override
+ public Config getDesiredConfigByType(String configType, @Nullable Map<String, DesiredConfig> desiredConfigs) {
+ DesiredConfig desiredConfig = (desiredConfigs == null) ? null : desiredConfigs.get(configType);
+ return (desiredConfig == null)
+ ? getDesiredConfigByType(configType)
+ : getConfig(configType, desiredConfig.getTag());
+ }
+
@Override
public List<Config> getLatestConfigsWithTypes(Collection<String> types) {
return clusterDAO.getLatestConfigurationsWithTypes(clusterId, getDesiredStackVersion(), types)
@@ -1522,13 +1530,8 @@ public class ClusterImpl implements Cluster {
public Map<String, DesiredConfig> getDesiredConfigs(boolean cachedConfigEntities) {
Map<String, Set<DesiredConfig>> activeConfigsByType = getDesiredConfigs(false, cachedConfigEntities);
return Maps.transformEntries(
- activeConfigsByType,
- new Maps.EntryTransformer<String, Set<DesiredConfig>, DesiredConfig>() {
- @Override
- public DesiredConfig transformEntry(@Nullable String key, @Nullable Set<DesiredConfig> value) {
- return value.iterator().next();
- }
- });
+ activeConfigsByType,
+ (key, value) -> value.iterator().next());
}
/**
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 7a3069b8d8..d8d9131372 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@ -34,6 +34,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import javax.persistence.RollbackException;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.ambari.server.ClusterNotFoundException;
import org.apache.ambari.server.DuplicateResourceException;
import org.apache.ambari.server.HostNotFoundException;
@@ -321,7 +322,7 @@ public class ClustersImpl implements Clusters {
for (Long hostId : hostsById.keySet()) {
try {
m_agentConfigsHolder.get().initializeDataIfNeeded(hostId, true);
- } catch (AmbariException e) {
+ } catch (AmbariRuntimeException e) {
LOG.error("Agent configs initialization was failed", e);
}
}
@@ -500,7 +501,7 @@ public class ClustersImpl implements Clusters {
// init host configs
try {
m_agentConfigsHolder.get().initializeDataIfNeeded(hostId, true);
- } catch (AmbariException e) {
+ } catch (AmbariRuntimeException e) {
LOG.error("Agent configs initialization was failed for host with id %s", hostId, e);
}
}
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 52b9a42315..8702001c1a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -29,6 +29,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.HostNotFoundException;
@@ -1105,9 +1106,8 @@ public class HostImpl implements Host {
@Override
public Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster,
Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException {
- Map<String, HostConfig> hostConfigMap = new HashMap<>();
- if( null == cluster ){
+ if(null == cluster){
clusterDesiredConfigs = new HashMap<>();
}
@@ -1116,42 +1116,36 @@ public class HostImpl implements Host {
clusterDesiredConfigs = cluster.getDesiredConfigs();
}
- if (clusterDesiredConfigs != null) {
- for (Map.Entry<String, DesiredConfig> desiredConfigEntry
- : clusterDesiredConfigs.entrySet()) {
+ Map<String, HostConfig> hostConfigMap = clusterDesiredConfigs.entrySet().stream().collect(Collectors.toMap(
+ Map.Entry::getKey,
+ desiredConfigEntry -> {
HostConfig hostConfig = new HostConfig();
hostConfig.setDefaultVersionTag(desiredConfigEntry.getValue().getTag());
- hostConfigMap.put(desiredConfigEntry.getKey(), hostConfig);
+ return hostConfig;
}
- }
+ ));
Map<Long, ConfigGroup> configGroups = (cluster == null) ? new HashMap<>() : cluster.getConfigGroupsByHostname(getHostName());
-
- if (configGroups != null && !configGroups.isEmpty()) {
- for (ConfigGroup configGroup : configGroups.values()) {
- for (Map.Entry<String, Config> configEntry : configGroup
- .getConfigurations().entrySet()) {
-
- String configType = configEntry.getKey();
- // HostConfig config holds configType -> versionTag, per config group
- HostConfig hostConfig = hostConfigMap.get(configType);
- if (hostConfig == null) {
- hostConfig = new HostConfig();
- hostConfigMap.put(configType, hostConfig);
- if (cluster != null) {
- Config conf = cluster.getDesiredConfigByType(configType);
- if(conf == null) {
- LOG.error("Config inconsistency exists:"+
- " unknown configType="+configType);
- } else {
- hostConfig.setDefaultVersionTag(conf.getTag());
- }
- }
+ if (configGroups == null || configGroups.isEmpty()) {
+ return hostConfigMap;
+ }
+
+ for (ConfigGroup configGroup : configGroups.values()) {
+ for (Map.Entry<String, Config> configEntry : configGroup.getConfigurations().entrySet()) {
+ String configType = configEntry.getKey();
+ // HostConfig config holds configType -> versionTag, per config group
+ HostConfig hostConfig = hostConfigMap.get(configType);
+ if (hostConfig == null) {
+ hostConfig = new HostConfig();
+ hostConfigMap.put(configType, hostConfig);
+ Config conf = cluster.getDesiredConfigByType(configType);
+ if(conf == null) {
+ LOG.error("Config inconsistency exists: unknown configType=" + configType);
+ } else {
+ hostConfig.setDefaultVersionTag(conf.getTag());
}
- Config config = configEntry.getValue();
- hostConfig.getConfigGroupOverrides().put(configGroup.getId(),
- config.getTag());
}
+ hostConfig.getConfigGroupOverrides().put(configGroup.getId(), configEntry.getValue().getTag());
}
}
return hostConfigMap;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
index bf2539ea19..2c8c3b8487 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelper.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,14 +86,14 @@ public class VariableReplacementHelper {
* @param replacementsMap a Map of data used to perform the variable replacements
* @return a new String
*/
- public String replaceVariables(String value, Map<String, Map<String, String>> replacementsMap) throws AmbariException {
+ public String replaceVariables(String value, Map<String, Map<String, String>> replacementsMap) throws AmbariRuntimeException {
if ((value != null) && (replacementsMap != null) && !replacementsMap.isEmpty()) {
int count = 0; // Used to help prevent an infinite loop...
boolean replacementPerformed;
do {
if (++count > 1000) {
- throw new AmbariException(String.format("Circular reference found while replacing variables in %s", value));
+ throw new AmbariRuntimeException(String.format("Circular reference found while replacing variables in %s", value));
}
Matcher matcher = PATTERN_VARIABLE.matcher(value);
@@ -106,35 +106,26 @@ public class VariableReplacementHelper {
String name = matcher.group(2);
String function = matcher.group(3);
- Map<String, String> replacements;
-
if ((name != null) && !name.isEmpty()) {
- if (type == null) {
- replacements = replacementsMap.get("");
- } else {
- replacements = replacementsMap.get(type);
+ Map<String, String> replacements = (type == null) ? replacementsMap.get("") : replacementsMap.get(type);
+ if (replacements == null || replacements.get(name) == null) {
+ continue;
}
- if (replacements != null) {
- String replacement = replacements.get(name);
-
- if (replacement != null) {
- if (function != null) {
- replacement = applyReplacementFunction(function, replacement, replacementsMap);
- }
-
- // Escape '$' and '\' so they don't cause any issues.
- matcher.appendReplacement(sb, replacement.replace("\\", "\\\\").replace("$", "\\$"));
- replacementPerformed = true;
- }
+ String replacement = replacements.get(name);
+ if (function != null) {
+ replacement = applyReplacementFunction(function, replacement, replacementsMap);
}
+
+ // Escape '$' and '\' so they don't cause any issues.
+ matcher.appendReplacement(sb, replacement.replace("\\", "\\\\").replace("$", "\\$"));
+ replacementPerformed = true;
}
}
matcher.appendTail(sb);
value = sb.toString();
- }
- while (replacementPerformed); // Process the string again to make sure new variables were not introduced
+ } while (replacementPerformed); // Process the string again to make sure new variables were not introduced
}
return value;
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
index b8fa5c35b6..8031b0cd09 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
@@ -1619,7 +1619,7 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog {
Map<String, Map<String, String>> kerberosConfigurations = new HashMap<>();
Map<String, Set<String>> propertiesToIgnore = new HashMap<>();
List<ServiceComponentHost> schToProcess = kerberosHelper.getServiceComponentHostsToProcess(cluster, kerberosDescriptor, null, null);
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false);
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false, null);
boolean includeAmbariIdentity = true;
String dataDirectory = kerberosHelper.createTemporaryDirectory().getAbsolutePath();
try {
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/ThreadPools.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/ThreadPools.java
new file mode 100644
index 0000000000..ac148be4bb
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/ThreadPools.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.utils;
+
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadFactory;
+import java.util.stream.Collectors;
+
+
+import org.apache.ambari.server.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class ThreadPools {
+
+ /**
+ * Determines the need of waiting for all tasks to be processed:
+ * true - next task would be processed
+ * false - next task would be cancelled
+ * @param <T>
+ */
+ public interface ThreadPoolFutureResult<T> {
+ Boolean waitForNextTask(T taskResult);
+ }
+
+ private static final String AGENT_COMMAND_PUBLISHER_POOL_NAME = "agent-command-publisher";
+ private static final String DEFAULT_FORK_JOIN_POOL_NAME = "default-fork-join-pool";
+
+ private static final Logger LOG = LoggerFactory.getLogger(ThreadPools.class);
+
+ private final Configuration configuration;
+
+ private ForkJoinPool agentPublisherCommandsPool;
+ private ForkJoinPool defaultForkJoinPool;
+
+ @Inject
+ public ThreadPools(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ private void logThreadPoolCreation(String name, int size) {
+ LOG.info(String.format("Creating '%s' thread pool with configured size %d", name, size));
+ }
+
+ private ForkJoinPool.ForkJoinWorkerThreadFactory createNamedFactory(String name) {
+ return pool -> {
+ ForkJoinWorkerThread worker = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool);
+ worker.setName(name + "-" + worker.getPoolIndex());
+ return worker;
+ };
+ }
+
+ private Boolean forkJoinPoolShutdown(ForkJoinPool pool, boolean forced) {
+ if (pool == null) {
+ return true;
+ }
+ if (forced) {
+ pool.shutdownNow();
+ } else {
+ pool.shutdown();
+ }
+ return pool.isShutdown();
+ }
+
+ public ForkJoinPool getAgentPublisherCommandsPool() {
+ if (agentPublisherCommandsPool == null){
+ logThreadPoolCreation(AGENT_COMMAND_PUBLISHER_POOL_NAME, configuration.getAgentCommandPublisherThreadPoolSize());
+ agentPublisherCommandsPool = new ForkJoinPool(
+ configuration.getAgentCommandPublisherThreadPoolSize(),
+ createNamedFactory(AGENT_COMMAND_PUBLISHER_POOL_NAME),
+ null,
+ false
+ );
+ }
+ return agentPublisherCommandsPool;
+ }
+
+ public ForkJoinPool getDefaultForkJoinPool() {
+ if (defaultForkJoinPool == null){
+ logThreadPoolCreation(DEFAULT_FORK_JOIN_POOL_NAME, configuration.getDefaultForkJoinPoolSize());
+ defaultForkJoinPool = new ForkJoinPool(
+ configuration.getDefaultForkJoinPoolSize(),
+ createNamedFactory(DEFAULT_FORK_JOIN_POOL_NAME),
+ null,
+ false
+ );
+ }
+ return defaultForkJoinPool;
+ }
+
+ public void shutdownDefaultForkJoinPool(boolean force){
+ if (forkJoinPoolShutdown(defaultForkJoinPool, force)) {
+ defaultForkJoinPool = null;
+ }
+ }
+
+ public void shutdownAgentPublisherCommandsPool(boolean force){
+ if (forkJoinPoolShutdown(agentPublisherCommandsPool, force)) {
+ agentPublisherCommandsPool = null;
+ }
+ }
+
+ /**
+ * Run {@code task} in parallel
+ *
+ * @param factoryName name of thread pool
+ * @param threadPoolSize maximum amount of threads to use
+ * @param operation operation caption
+ * @param tasks list of callables to be submitted to thread pool
+ * @param taskResultFunc process result of each task with
+ * @param <T> return type of task callable
+ * @throws Exception
+ */
+ public <T> void parallelOperation(String factoryName, int threadPoolSize, String operation,
+ List<Callable<T>> tasks, ThreadPoolFutureResult<T> taskResultFunc)
+ throws Exception{
+
+ logThreadPoolCreation(factoryName, threadPoolSize);
+ ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat(factoryName).build();
+ ExecutorService executorService = Executors.newFixedThreadPool(threadPoolSize, threadFactory);
+ CompletionService<T> completionService = new ExecutorCompletionService<>(executorService);
+ List<Future<T>> futures = tasks.stream().map(completionService::submit).collect(Collectors.toList());
+
+ LOG.info("Processing {} {} concurrently...", futures.size(), operation);
+ T t;
+ try {
+ for( int i = 0; i < futures.size(); i++ ) {
+ Future<T> future = completionService.take();
+ t = future.get();
+
+ if (!taskResultFunc.waitForNextTask(t)){
+ break;
+ }
+ }
+ } finally {
+ futures.stream()
+ .filter(x -> !x.isCancelled() && !x.isDone())
+ .forEach(x -> x.cancel(true));
+
+ executorService.shutdown();
+ }
+ }
+
+ public static ExecutorService getSingleThreadedExecutor(String threadPoolName) {
+ return Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat(threadPoolName + "-%d")
+ .build()
+ );
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ shutdownAgentPublisherCommandsPool(true);
+ shutdownDefaultForkJoinPool(true);
+
+ super.finalize();
+ }
+}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index 2e43853591..7017b4a61c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -1483,10 +1483,10 @@ public class TestHeartbeatHandler {
replay(am);
Method injectKeytabMethod = agentCommandsPublisher.getClass().getDeclaredMethod("injectKeytab",
- ExecutionCommand.class, String.class, String.class);
+ ExecutionCommand.class, String.class, String.class, Map.class);
injectKeytabMethod.setAccessible(true);
commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData(agentCommandsPublisher, false).getAbsolutePath());
- injectKeytabMethod.invoke(agentCommandsPublisher, executionCommand, "SET_KEYTAB", targetHost);
+ injectKeytabMethod.invoke(agentCommandsPublisher, executionCommand, "SET_KEYTAB", targetHost, null);
return executionCommand.getKerberosCommandParams();
}
@@ -1516,10 +1516,10 @@ public class TestHeartbeatHandler {
replay(am);
Method injectKeytabMethod = agentCommandsPublisher.getClass().getDeclaredMethod("injectKeytab",
- ExecutionCommand.class, String.class, String.class);
+ ExecutionCommand.class, String.class, String.class, Map.class);
injectKeytabMethod.setAccessible(true);
commandparams.put(KerberosServerAction.DATA_DIRECTORY, createTestKeytabData(agentCommandsPublisher, true).getAbsolutePath());
- injectKeytabMethod.invoke(agentCommandsPublisher, executionCommand, "REMOVE_KEYTAB", targetHost);
+ injectKeytabMethod.invoke(agentCommandsPublisher, executionCommand, "REMOVE_KEYTAB", targetHost, null);
return executionCommand.getKerberosCommandParams();
}
@@ -1577,7 +1577,7 @@ public class TestHeartbeatHandler {
)
).once();
- expect(kerberosKeytabControllerMock.getServiceIdentities(EasyMock.anyString(), EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
+ expect(kerberosKeytabControllerMock.getServiceIdentities(EasyMock.anyString(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
replay(kerberosKeytabControllerMock);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index 9fcb38af62..72033f21d4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -1106,7 +1106,7 @@ public class KerberosHelperTest extends EasyMockSupport {
}
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -1266,7 +1266,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.once();
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -1465,7 +1465,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.once();
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -2525,6 +2525,7 @@ public class KerberosHelperTest extends EasyMockSupport {
final Cluster cluster = createMockCluster("c1", hosts, SecurityType.KERBEROS, krb5ConfConfig, kerberosEnvConfig);
expect(cluster.getServices()).andReturn(services).anyTimes();
expect(cluster.getServiceComponentHostMap(EasyMock.anyObject(), EasyMock.anyObject())).andReturn(serviceComponentHostMap).anyTimes();
+ expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.2")).anyTimes();
final Map<String, Map<String, String>> existingConfigurations = new HashMap<String, Map<String, String>>() {
{
@@ -2763,6 +2764,7 @@ public class KerberosHelperTest extends EasyMockSupport {
Cluster cluster = createMockCluster(clusterName, Arrays.asList(host1, host2, host3), SecurityType.KERBEROS, configKrb5Conf, configKerberosEnv);
expect(cluster.getServices()).andReturn(servicesMap).anyTimes();
+ expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.2")).anyTimes();
Map<String, String> kerberosDescriptorProperties = new HashMap<>();
kerberosDescriptorProperties.put("additional_realms", "");
@@ -2897,7 +2899,7 @@ public class KerberosHelperTest extends EasyMockSupport {
}
CreateKeytabFilesServerAction createKeytabFilesServerAction = injector.getInstance(CreateKeytabFilesServerAction.class);
- expect(createKeytabFilesServerAction.createKeytab(capture(capturePrincipalForKeytab), eq("password"), eq(1), anyObject(KerberosOperationHandler.class), eq(true), eq(true), isNull(ActionLog.class)))
+ expect(createKeytabFilesServerAction.createKeytab(capture(capturePrincipalForKeytab),anyObject(), eq("password"), eq(1), anyObject(KerberosOperationHandler.class), eq(true), eq(true), isNull(ActionLog.class)))
.andReturn(new Keytab())
.times(createAmbariIdentities ? 5 : 3);
@@ -2976,6 +2978,7 @@ public class KerberosHelperTest extends EasyMockSupport {
servicesMap.put("SERVICE1", service1);
Cluster cluster = createMockCluster("c1", Collections.singletonList(host1), SecurityType.KERBEROS, configKrb5Conf, configKerberosEnv);
+ expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.2")).anyTimes();
expect(cluster.getServices()).andReturn(servicesMap).anyTimes();
Map<String, String> kerberosDescriptorProperties = new HashMap<>();
@@ -3249,7 +3252,7 @@ public class KerberosHelperTest extends EasyMockSupport {
}
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.times(3);
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -3460,7 +3463,7 @@ public class KerberosHelperTest extends EasyMockSupport {
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -3570,7 +3573,7 @@ public class KerberosHelperTest extends EasyMockSupport {
result.created = true;
result.kkp = kkp;
- expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject())).andReturn(result).anyTimes();
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject(), anyObject())).andReturn(result).anyTimes();
boolean managingIdentities = !Boolean.FALSE.equals(manageIdentities);
final Map<String, String> kerberosEnvProperties = new HashMap<>();
@@ -3677,7 +3680,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.anyTimes();
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -3685,7 +3688,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.once();
final ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
- expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), EasyMock.anyObject()))
+ expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), anyObject(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>() {
{
put("cluster-env", new HashMap<String, String>() {{
@@ -3765,7 +3768,7 @@ public class KerberosHelperTest extends EasyMockSupport {
result.created = true;
result.kkp = kkp;
- expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject())).andReturn(result).anyTimes();
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), anyObject(), anyObject(), anyObject())).andReturn(result).anyTimes();
Host host1 = createMock(Host.class);
expect(host1.getHostId()).andReturn(1l).anyTimes();
@@ -3855,7 +3858,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.once();
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.once();
expect(ambariManagementController.getRoleCommandOrder(cluster))
@@ -3863,7 +3866,7 @@ public class KerberosHelperTest extends EasyMockSupport {
.once();
final ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
- expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), EasyMock.anyObject()))
+ expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), anyObject(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>() {
{
put("cluster-env", new HashMap<String, String>() {
@@ -4000,6 +4003,8 @@ public class KerberosHelperTest extends EasyMockSupport {
expect(cluster.getSecurityType()).andReturn(clusterSecurityType).anyTimes();
expect(cluster.getClusterName()).andReturn(clusterName).anyTimes();
expect(cluster.getClusterId()).andReturn(2L).anyTimes();
+ expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.2")).anyTimes();
+ expect(cluster.getDesiredConfigByType(anyString(), anyObject())).andReturn(EasyMock.createNiceMock(Config.class)).anyTimes();
expect(cluster.getServiceComponentHosts("host1"))
.andReturn(new ArrayList<ServiceComponentHost>() {
{
@@ -4074,18 +4079,18 @@ public class KerberosHelperTest extends EasyMockSupport {
}
final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1"))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host1", null))
.andReturn(Collections.emptyMap())
.anyTimes();
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host2"))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, "host2", null))
.andReturn(Collections.emptyMap())
.anyTimes();
- expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(Collections.emptyMap())
.anyTimes();
final ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
- expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), EasyMock.anyObject()))
+ expect(configHelper.calculateExistingConfigurations(eq(ambariManagementController), anyObject(Cluster.class), anyObject(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>() {
{
put("cluster-env", new HashMap<String, String>() {{
@@ -4187,6 +4192,7 @@ public class KerberosHelperTest extends EasyMockSupport {
expect(kerberosDescriptor.getService("KERBEROS")).andReturn(null).anyTimes();
expect(kerberosDescriptor.getService("SERVICE1")).andReturn(serviceDescriptor1).anyTimes();
expect(kerberosDescriptor.getService("SERVICE2")).andReturn(serviceDescriptor2).anyTimes();
+ expect(kerberosDescriptor.getService("AMBARI")).andReturn(null).anyTimes();
setupKerberosDescriptor(kerberosDescriptor);
@@ -4197,7 +4203,7 @@ public class KerberosHelperTest extends EasyMockSupport {
Map<String, Collection<KerberosIdentityDescriptor>> identities;
identities = kerberosHelper.getActiveIdentities(clusterName, hostName, serviceName,
- componentName, replaceHostNames, null, null);
+ componentName, replaceHostNames, null, null, null);
verifyAll();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProviderTest.java
index e867af332e..691e4cae7f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterKerberosDescriptorResourceProviderTest.java
@@ -353,11 +353,11 @@ public class ClusterKerberosDescriptorResourceProviderTest extends EasyMockSuppo
compositeKerberosDescriptor.update(userKerberosDescriptor);
KerberosHelper kerberosHelper = createMock(KerberosHelper.class);
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.STACK), eq(cluster), eq(false), anyObject(Collection.class), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.STACK), eq(cluster), eq(false), anyObject(Collection.class), eq(false), anyObject(), anyObject()))
.andReturn(stackKerberosDescriptor).atLeastOnce();
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.USER), eq(cluster), eq(false), anyObject(Collection.class), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.USER), eq(cluster), eq(false), anyObject(Collection.class), eq(false), anyObject(), anyObject()))
.andReturn(userKerberosDescriptor).atLeastOnce();
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.COMPOSITE), eq(cluster), eq(false), anyObject(Collection.class), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.COMPOSITE), eq(cluster), eq(false), anyObject(Collection.class), eq(false), anyObject(), anyObject()))
.andReturn(compositeKerberosDescriptor).atLeastOnce();
AmbariManagementController managementController = createMock(AmbariManagementController.class);
@@ -465,11 +465,11 @@ public class ClusterKerberosDescriptorResourceProviderTest extends EasyMockSuppo
Capture<? extends Collection<String>> captureAdditionalServices = newCapture(CaptureType.ALL);
KerberosHelper kerberosHelper = createMock(KerberosHelper.class);
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.STACK), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.STACK), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false), anyObject(), anyObject()))
.andReturn(stackKerberosDescriptor).atLeastOnce();
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.USER), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.USER), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false), anyObject(), anyObject()))
.andReturn(userKerberosDescriptor).atLeastOnce();
- expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.COMPOSITE), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false)))
+ expect(kerberosHelper.getKerberosDescriptor(eq(KerberosHelper.KerberosDescriptorType.COMPOSITE), eq(cluster), eq(true), capture(captureAdditionalServices), eq(false), anyObject(), anyObject()))
.andReturn(compositeKerberosDescriptor).atLeastOnce();
AmbariManagementController managementController = createMock(AmbariManagementController.class);
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
index 6aa6fbab48..03f19abb33 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java
@@ -298,7 +298,7 @@ public class ClusterStackVersionResourceProviderTest {
expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class),
EasyMock.<Map<String, String>>anyObject(), anyObject(String.class))).
andReturn(packages).anyTimes();
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster);
@@ -719,7 +719,7 @@ public class ClusterStackVersionResourceProviderTest {
expect(resourceProviderFactory.getHostResourceProvider(
eq(managementController))).andReturn(csvResourceProvider).anyTimes();
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
@@ -943,7 +943,7 @@ public class ClusterStackVersionResourceProviderTest {
EasyMock.anyObject(), anyObject(String.class))).
andReturn(packages).anyTimes(); // only one host has the versionable component
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster);
@@ -1157,7 +1157,7 @@ public class ClusterStackVersionResourceProviderTest {
EasyMock.<Map<String, String>>anyObject(), anyObject(String.class))).
andReturn(packages).anyTimes();
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster);
@@ -1534,7 +1534,7 @@ public class ClusterStackVersionResourceProviderTest {
(Map<String, String>) anyObject(List.class), anyObject(String.class))).
andReturn(packages).anyTimes();
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
@@ -1958,7 +1958,7 @@ public class ClusterStackVersionResourceProviderTest {
EasyMock.<Map<String, String>>anyObject(), anyObject(String.class))).
andReturn(packages).anyTimes(); // only one host has the versionable component
- expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString()))
+ expect(managementController.findConfigurationTagsWithOverrides(anyObject(Cluster.class), EasyMock.anyString(), anyObject()))
.andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster).anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
index c9143a96ca..bde0adea42 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostKerberosIdentityResourceProviderTest.java
@@ -228,7 +228,7 @@ public class HostKerberosIdentityResourceProviderTest extends EasyMockSupport {
EasyMock.eq(false))).andReturn(kerberosDescriptor).atLeastOnce();
expect(kerberosHelper.getActiveIdentities("Cluster100", "Host100", null, null, true, null,
- kerberosDescriptor))
+ kerberosDescriptor, null))
.andReturn(activeIdentities)
.once();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
index 8370772c7f..ee6660c1c2 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/RestMetricsPropertyProviderTest.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.controller.metrics;
+import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.eq;
@@ -160,7 +161,7 @@ public class RestMetricsPropertyProviderTest {
ConfigHelper configHelperMock = createNiceMock(ConfigHelper.class);
expect(amc.getClusters()).andReturn(clusters).anyTimes();
expect(amc.getAmbariEventPublisher()).andReturn(createNiceMock(AmbariEventPublisher.class)).anyTimes();
- expect(amc.findConfigurationTagsWithOverrides(eq(c1), anyString())).andReturn(Collections.singletonMap("storm-site",
+ expect(amc.findConfigurationTagsWithOverrides(eq(c1), anyString(), anyObject())).andReturn(Collections.singletonMap("storm-site",
Collections.singletonMap("tag", "version1"))).anyTimes();
expect(amc.getConfigHelper()).andReturn(configHelperMock).anyTimes();
expect(configHelperMock.getEffectiveConfigProperties(eq(c1),
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerActionTest.java
index 1eaf594cd3..8bfe2fe56e 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerActionTest.java
@@ -60,6 +60,7 @@ import org.apache.ambari.server.ldap.service.AmbariLdapConfigurationProvider;
import org.apache.ambari.server.metadata.RoleCommandOrderProvider;
import org.apache.ambari.server.mpack.MpackManagerFactory;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.orm.dao.KerberosKeytabPrincipalDAO;
import org.apache.ambari.server.scheduler.ExecutionScheduler;
import org.apache.ambari.server.security.encryption.CredentialStoreService;
import org.apache.ambari.server.security.encryption.Encryptor;
@@ -234,6 +235,7 @@ public class AbstractPrepareKerberosServerActionTest extends EasyMockSupport {
bind(EntityManager.class).toProvider(entityManagerProvider);
bind(MpackManagerFactory.class).toInstance(createNiceMock(MpackManagerFactory.class));
bind(AmbariLdapConfigurationProvider.class).toInstance(createMock(AmbariLdapConfigurationProvider.class));
+ bind(KerberosKeytabPrincipalDAO.class).toInstance(createNiceMock(KerberosKeytabPrincipalDAO.class));
bind(new TypeLiteral<Encryptor<AgentConfigsUpdateEvent>>() {}).annotatedWith(Names.named("AgentConfigEncryptor")).toInstance(Encryptor.NONE);
bind(new TypeLiteral<Encryptor<AmbariServerConfiguration>>() {}).annotatedWith(Names.named("AmbariServerConfigurationEncryptor")).toInstance(Encryptor.NONE);
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
index 191b6cbe37..78e3592a7f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/ConfigureAmbariIdentitiesServerActionTest.java
@@ -101,7 +101,7 @@ public class ConfigureAmbariIdentitiesServerActionTest extends EasyMockSupport {
result.created = true;
result.kkp = kkp;
- expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), eq(hostEntity), anyObject())).andReturn(result).once();
+ expect(kerberosKeytabPrincipalDAO.findOrCreate(anyObject(), eq(hostEntity), anyObject(), anyObject())).andReturn(result).once();
expect(kerberosKeytabPrincipalDAO.merge(kkp)).andReturn(createNiceMock(KerberosKeytabPrincipalEntity.class)).once();
// Mock the methods that do the actual file manipulation to avoid having to deal with ambari-sudo.sh used in
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
index e49ecfd455..4c46728257 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerActionTest.java
@@ -276,7 +276,7 @@ public class KerberosServerActionTest extends EasyMockSupport {
expectLastCall().atLeastOnce();
KerberosOperationHandlerFactory factory = injector.getInstance(KerberosOperationHandlerFactory.class);
- expect(factory.getKerberosOperationHandler(KDCType.MIT_KDC)).andReturn(kerberosOperationHandler).once();
+ expect(factory.getKerberosOperationHandler(KDCType.MIT_KDC)).andReturn(kerberosOperationHandler).anyTimes();
replayAll();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java
index 9850051e21..fccf4fec3a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/kerberos/UpdateKerberosConfigsServerActionTest.java
@@ -25,9 +25,9 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.stream.StreamSupport;
import javax.persistence.EntityManager;
@@ -40,7 +40,6 @@ import org.apache.ambari.server.stack.StackManagerFactory;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
-import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.cluster.ClustersImpl;
import org.apache.ambari.server.state.stack.OsFamily;
import org.apache.ambari.server.testutils.PartialNiceMockBinder;
@@ -119,8 +118,8 @@ public class UpdateKerberosConfigsServerActionTest extends EasyMockSupport{
executionCommand.setCommandParams(commandParams);
ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
- configHelper.updateConfigType(anyObject(Cluster.class), anyObject(StackId.class), anyObject(AmbariManagementController.class),
- anyObject(String.class), EasyMock.anyObject(), EasyMock.anyObject(), anyObject(String.class), anyObject(String.class));
+ configHelper.updateBulkConfigType(anyObject(), anyObject(), anyObject(), anyObject(), anyObject(), anyObject(),
+ anyObject(), anyObject());
expectLastCall().atLeastOnce();
replayAll();
@@ -161,6 +160,7 @@ public class UpdateKerberosConfigsServerActionTest extends EasyMockSupport{
}
@Test
+ @SuppressWarnings("unchecked")
public void testUpdateConfigForceSecurityEnabled() throws Exception {
Map<String, String> commandParams = new HashMap<>();
commandParams.put(KerberosServerAction.DATA_DIRECTORY, dataDir);
@@ -170,10 +170,19 @@ public class UpdateKerberosConfigsServerActionTest extends EasyMockSupport{
ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
- Capture<String> configTypes = Capture.newInstance(CaptureType.ALL);
- Capture<Map<String, String>> configUpdates = Capture.newInstance(CaptureType.ALL);
- configHelper.updateConfigType(anyObject(Cluster.class), anyObject(StackId.class), anyObject(AmbariManagementController.class),
- capture(configTypes), capture(configUpdates), anyObject(Collection.class), anyObject(String.class), anyObject(String.class));
+ Capture<Iterable<String>> configTypes = Capture.newInstance(CaptureType.ALL);
+ Capture<Map<String, Map<String, String>>> configUpdates = Capture.newInstance(CaptureType.ALL);
+
+ configHelper.updateBulkConfigType(
+ anyObject(),
+ anyObject(),
+ anyObject(),
+ capture(configTypes),
+ capture(configUpdates),
+ anyObject(),
+ anyObject(),
+ anyObject()
+ );
expectLastCall().atLeastOnce();
replayAll();
@@ -181,15 +190,17 @@ public class UpdateKerberosConfigsServerActionTest extends EasyMockSupport{
action.setExecutionCommand(executionCommand);
action.execute(null);
- assertTrue(configTypes.getValues().contains("cluster-env"));
- boolean containsSecurityEnabled = false;
- for(Map<String, String> properties: configUpdates.getValues()) {
- if(properties.containsKey("security_enabled")) {
- containsSecurityEnabled = true;
- break;
- }
- }
- assertTrue(containsSecurityEnabled);
+ assertTrue(StreamSupport.stream(configTypes.getValues().get(0).spliterator(), false).anyMatch(
+ config -> config.equals("cluster-env")
+ ));
+
+ assertTrue(
+ configUpdates.getValues().stream()
+ .flatMap(x -> x.values().stream())
+ .flatMap(x -> x.entrySet().stream())
+ .anyMatch(property -> property.getKey().equals("security_enabled"))
+ );
+
verifyAll();
}
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
index 7a1aae208f..ba7f7ff9fa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosActionTest.java
@@ -352,7 +352,7 @@ public class PreconfigureKerberosActionTest extends EasyMockSupport {
.andReturn(createMockServiceInfo("KNOX", knoxProperties, Collections.singletonList(createMockComponentInfo("KNOX_GATEWAY")))).anyTimes();
AmbariManagementController managementController = injector.getInstance(AmbariManagementController.class);
- expect(managementController.findConfigurationTagsWithOverrides(cluster, null))
+ expect(managementController.findConfigurationTagsWithOverrides(cluster, null, null))
.andReturn(clusterConfig).once();
expect(managementController.getAuthName()).andReturn("admin").anyTimes();
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
index 5152bee8c7..2c8e12af66 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/VariableReplacementHelperTest.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.AmbariRuntimeException;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -131,8 +132,8 @@ public class VariableReplacementHelperTest {
try {
assertEquals("${config-type2/self_reference}",
helper.replaceVariables("${config-type2/self_reference}", configurations));
- Assert.fail(String.format("%s expected to be thrown", AmbariException.class.getName()));
- } catch (AmbariException e) {
+ Assert.fail(String.format("%s expected to be thrown", AmbariRuntimeException.class.getName()));
+ } catch (AmbariRuntimeException e) {
// This is expected...
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org