You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2017/09/28 13:25:34 UTC
[45/50] [abbrv] ambari git commit: Merge remote-tracking branch
'remotes/origin/trunk' into branch-3.0-perf
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
index 0000000,30bc47f..697f1d1
mode 000000,100644..100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/PreconfigureKerberosAction.java
@@@ -1,0 -1,573 +1,573 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.ambari.server.serveraction.upgrades;
+
+ import static org.apache.ambari.server.controller.KerberosHelper.DEFAULT_REALM;
+ import static org.apache.ambari.server.controller.KerberosHelper.KERBEROS_ENV;
+ import static org.apache.ambari.server.controller.KerberosHelper.PRECONFIGURE_SERVICES;
+
+ import java.io.IOException;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+ import java.util.concurrent.ConcurrentMap;
+
+ import org.apache.ambari.server.AmbariException;
+ import org.apache.ambari.server.actionmanager.HostRoleStatus;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.controller.AmbariManagementController;
+ import org.apache.ambari.server.controller.KerberosHelper;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+ import org.apache.ambari.server.serveraction.kerberos.PreconfigureServiceType;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.ConfigHelper;
+ import org.apache.ambari.server.state.Host;
+ import org.apache.ambari.server.state.SecurityType;
+ import org.apache.ambari.server.state.Service;
+ import org.apache.ambari.server.state.ServiceComponentHost;
+ import org.apache.ambari.server.state.StackId;
+ import org.apache.ambari.server.state.UpgradeContext;
+ import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
+ import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
+ import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+ import org.apache.ambari.server.state.kerberos.VariableReplacementHelper;
+ import org.apache.ambari.server.state.stack.upgrade.Direction;
+ import org.apache.commons.collections.CollectionUtils;
+ import org.apache.commons.collections.MapUtils;
+ import org.apache.commons.lang.StringUtils;
+
+ import com.google.inject.Inject;
+
+ /**
+ * PreconfigureKerberos updates existing service configurations with properties from service-level
+ * Kerberos descriptors, flagged for pre-configuring, during stack upgrades in order to prevent service
+ * restarts when the flagged services are installed.
+ */
+ public class PreconfigureKerberosAction extends AbstractUpgradeServerAction {
+ static final String UPGRADE_DIRECTION_KEY = "upgrade_direction";
+
+ @Inject
+ private AmbariManagementController ambariManagementController;
+
+ @Inject
+ private KerberosHelper kerberosHelper;
+
+ @Inject
+ private ConfigHelper configHelper;
+
+ @Inject
+ private VariableReplacementHelper variableReplacementHelper;
+
+ @Override
+ public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext) throws AmbariException, InterruptedException {
+ Map<String, String> commandParameters = getCommandParameters();
+ if (null == commandParameters || commandParameters.isEmpty()) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+ "Unable to change configuration values without command parameters");
+ }
+
+ if (!isDowngrade()) {
+ String clusterName = commandParameters.get("clusterName");
+ Cluster cluster = m_clusters.getCluster(clusterName);
+
+ if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+ StackId stackId;
+
+ try {
+ stackId = getTargetStackId(cluster);
+ } catch (AmbariException e) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "", e.getLocalizedMessage());
+ }
+
+ if (stackId == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED, "{}", "",
+ "The target stack Id was not specified.");
+ }
+
+ KerberosDescriptor kerberosDescriptor = kerberosHelper.getKerberosDescriptor(KerberosHelper.KerberosDescriptorType.COMPOSITE, cluster, stackId, true);
+
+ // Calculate the current host-specific configurations. These will be used to replace
+ // variables within the Kerberos descriptor data
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, true, false);
+
+ PreconfigureServiceType preconfigureServiceType = getPreconfigureServiceType(configurations);
+
+ if (preconfigureServiceType != PreconfigureServiceType.NONE) {
+ Map<String, Map<String, String>> kerberosConfigurations = new HashMap<>();
+ Map<String, Set<String>> propertiesToRemove = new HashMap<>();
+ Map<String, Set<String>> propertiesToIgnore = new HashMap<>();
+
+ if (preconfigureServiceType == PreconfigureServiceType.ALL) {
+ // Force all services to be flagged for pre-configuration...
+ Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices();
+ if (serviceDescriptors != null) {
+ for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) {
+ serviceDescriptor.setPreconfigure(true);
+ }
+ }
+ }
+
+ processServiceComponentHosts(cluster, kerberosDescriptor, configurations, kerberosConfigurations, propertiesToIgnore);
+
+ // Calculate the set of configurations to update and replace any variables
+ // using the previously calculated Map of configurations for the host.
+ kerberosConfigurations = kerberosHelper.processPreconfiguredServiceConfigurations(kerberosConfigurations, configurations, cluster, kerberosDescriptor);
+
+ Map<String, Set<String>> installedServices = calculateInstalledServices(cluster);
+
+ kerberosHelper.applyStackAdvisorUpdates(cluster, installedServices.keySet(), configurations, kerberosConfigurations,
+ propertiesToIgnore, propertiesToRemove, true);
+
+ kerberosHelper.setAuthToLocalRules(cluster, kerberosDescriptor, getDefaultRealm(configurations), installedServices,
+ configurations, kerberosConfigurations, true);
+
+ processConfigurationChanges(cluster, stackId, kerberosDescriptor, kerberosConfigurations, propertiesToRemove, configurations);
+ } else {
+ actionLog.writeStdOut("Skipping: This facility is only available when kerberos-env/preconfigure_services is not \"NONE\"");
+ }
+ } else {
+ actionLog.writeStdOut("Skipping: This facility is only available when Kerberos is enabled");
+ }
+ } else {
+ actionLog.writeStdOut("Skipping: This facility is only available during an upgrade");
+ }
+
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
+ }
+
+ /**
+ * Given a Cluster object creates a map of service names to sets of the installed components for that
+ * service.
+ *
+ * @param cluster the cluster
+ * @return a map of (installed) service names to the relevant set of (installed) component names
+ */
+ private Map<String, Set<String>> calculateInstalledServices(Cluster cluster) {
+ Map<String, Set<String>> installedServices = new HashMap<>();
+ Map<String, Service> services = cluster.getServices();
+
+ for (Service service : services.values()) {
+ installedServices.put(service.getName(), service.getServiceComponents().keySet());
+ }
+
+ return installedServices;
+ }
+
+ /**
+ * Safely retrieves the specified property from the specified configuration type from a map of
+ * configurations.
+ *
+ * @param configurations the existing configurations for the cluster
+ * @return the requested value or null if the configuration does not exist
+ */
+ private String getValueFromConfiguration(Map<String, Map<String, String>> configurations, String configType, String propertyName) {
+ String value = null;
+
+ if (configurations != null) {
+ Map<String, String> kerberosEnv = configurations.get(configType);
+
+ if (kerberosEnv != null) {
+ value = kerberosEnv.get(propertyName);
+ }
+ }
+
+ return value;
+ }
+
+ /**
+ * Safely retrieves the <code>realm</code> property of the <code>kerberos-env</code> configuration.
+ *
+ * @param configurations the existing configurations for the cluster
+ * @return the requested value or null if the configuration does not exist
+ * @see #getValueFromConfiguration(Map, String, String)
+ */
+ private String getDefaultRealm(Map<String, Map<String, String>> configurations) {
+ return getValueFromConfiguration(configurations, KERBEROS_ENV, DEFAULT_REALM);
+ }
+
+ /**
+ * Safely retrieves the <code>preconfigure_services</code> property of the <code>kerberos-env</code> configuration.
+ *
+ * @param configurations the existing configurations for the cluster
+ * @return the requested value or null if the configuration does not exist
+ * @see #getValueFromConfiguration(Map, String, String)
+ */
+ private PreconfigureServiceType getPreconfigureServiceType(Map<String, Map<String, String>> configurations) {
+ String preconfigureServices = getValueFromConfiguration(configurations, KERBEROS_ENV, PRECONFIGURE_SERVICES);
+
+ PreconfigureServiceType preconfigureServiceType = null;
+ if (!StringUtils.isEmpty(preconfigureServices)) {
+ try {
+ preconfigureServiceType = PreconfigureServiceType.valueOf(preconfigureServices.toUpperCase());
+ } catch (Throwable t) {
+ preconfigureServiceType = PreconfigureServiceType.DEFAULT;
+ }
+ }
+
+ return (preconfigureServiceType == null) ? PreconfigureServiceType.DEFAULT : preconfigureServiceType;
+ }
+
+ /**
+ * Determines if upgrade direction is {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}.
+ *
+ * @return {@code true} if {@link Direction#DOWNGRADE}; {@code false} if {@link Direction#UPGRADE}
+ */
+ private boolean isDowngrade() {
+ return Direction.DOWNGRADE.name().equalsIgnoreCase(getCommandParameterValue(UPGRADE_DIRECTION_KEY));
+ }
+
+ /**
+ * Retrieves the target stack ID for the stack upgrade or downgrade operation.
+ *
+ * @param cluster the cluster
+ * @return the target {@link StackId}
+ * @throws AmbariException if multiple stack id's are detected
+ */
+ private StackId getTargetStackId(Cluster cluster) throws AmbariException {
+ UpgradeContext upgradeContext = getUpgradeContext(cluster);
+
+ // !!! FIXME in a per-service view, what does this become?
+ Set<StackId> stackIds = new HashSet<>();
+
+ for (Service service : cluster.getServices().values()) {
+ RepositoryVersionEntity targetRepoVersion = upgradeContext.getTargetRepositoryVersion(service.getName());
+ StackId targetStackId = targetRepoVersion.getStackId();
+ stackIds.add(targetStackId);
+ }
+
+ if (1 != stackIds.size()) {
+ throw new AmbariException("Services are deployed from multiple stacks and cannot determine a unique one.");
+ }
+
+ return stackIds.iterator().next();
+ }
+
+ /**
+ * Find and iterate through the {@link ServiceComponentHost} objects for the current {@link Cluster}
+ * to calculate property updates and auth-to-local rules.
+ *
+ * @param cluster the cluster
+ * @param kerberosDescriptor the Kerberos descriptor
+ * @param currentConfigurations the current configurations for the cluster
+ * @param kerberosConfigurations the (Kerberos-specific) configuration updates
+ * @param propertiesToBeIgnored a map to store properties that should be ignored by operations that update property values
+ * @throws AmbariException if an issue occurs
+ */
+ private void processServiceComponentHosts(Cluster cluster, KerberosDescriptor kerberosDescriptor,
+ Map<String, Map<String, String>> currentConfigurations,
+ Map<String, Map<String, String>> kerberosConfigurations,
+ Map<String, Set<String>> propertiesToBeIgnored)
+ throws AmbariException {
+
+ Collection<Host> hosts = cluster.getHosts();
+ if (!hosts.isEmpty()) {
+ // Create the context to use for filtering Kerberos Identities based on the state of the cluster
+ Map<String, Object> filterContext = new HashMap<>();
+ filterContext.put("configurations", currentConfigurations);
+ filterContext.put("services", cluster.getServices().keySet());
+
+ try {
+ Map<String, Set<String>> propertiesToIgnore = null;
+
+ for (Host host : hosts) {
+ // Iterate over the components installed on the current host to get the service and
+ // component-level Kerberos descriptors in order to determine which principals,
+ // keytab files, and configurations need to be created or updated.
+ for (ServiceComponentHost sch : cluster.getServiceComponentHosts(host.getHostName())) {
+ String hostName = sch.getHostName();
+
+ String serviceName = sch.getServiceName();
+ String componentName = sch.getServiceComponentName();
+
+ KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName);
+
+ if (serviceDescriptor != null) {
+ List<KerberosIdentityDescriptor> serviceIdentities = serviceDescriptor.getIdentities(true, filterContext);
+
+ // Add service-level principals (and keytabs)
+ kerberosHelper.addIdentities(null, serviceIdentities,
- null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false);
++ null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations);
+ propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore);
+
+ KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName);
+
+ if (componentDescriptor != null) {
+ List<KerberosIdentityDescriptor> componentIdentities = componentDescriptor.getIdentities(true, filterContext);
+
+ // Calculate the set of configurations to update and replace any variables
+ // using the previously calculated Map of configurations for the host.
+ kerberosHelper.mergeConfigurations(kerberosConfigurations,
+ componentDescriptor.getConfigurations(true), currentConfigurations, null);
+
+ // Add component-level principals (and keytabs)
+ kerberosHelper.addIdentities(null, componentIdentities,
- null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations, false);
++ null, hostName, serviceName, componentName, kerberosConfigurations, currentConfigurations);
+ propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
+ }
+ }
+ }
+ }
+
+ // Add ambari-server identities only if 'kerberos-env.create_ambari_principal = true'
+ if (kerberosHelper.createAmbariIdentities(currentConfigurations.get(KERBEROS_ENV))) {
+ List<KerberosIdentityDescriptor> ambariIdentities = kerberosHelper.getAmbariServerIdentities(kerberosDescriptor);
+
+ for (KerberosIdentityDescriptor identity : ambariIdentities) {
+ // If the identity represents the ambari-server user, use the component name "AMBARI_SERVER_SELF"
+ // so it can be distinguished between other identities related to the AMBARI-SERVER
+ // component.
+ String componentName = KerberosHelper.AMBARI_SERVER_KERBEROS_IDENTITY_NAME.equals(identity.getName())
+ ? "AMBARI_SERVER_SELF"
+ : "AMBARI_SERVER";
+
+ List<KerberosIdentityDescriptor> componentIdentities = Collections.singletonList(identity);
+ kerberosHelper.addIdentities(null, componentIdentities,
- null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations, false);
++ null, KerberosHelper.AMBARI_SERVER_HOST_NAME, "AMBARI", componentName, kerberosConfigurations, currentConfigurations);
+ propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
+ }
+ }
+
+ if ((propertiesToBeIgnored != null) && (propertiesToIgnore != null)) {
+ propertiesToBeIgnored.putAll(propertiesToIgnore);
+ }
+ } catch (IOException e) {
+ throw new AmbariException(e.getMessage(), e);
+ }
+ }
+ }
+
+ private Map<String, Set<String>> gatherPropertiesToIgnore(List<KerberosIdentityDescriptor> identities,
+ Map<String, Set<String>> propertiesToIgnore) {
+ Map<String, Map<String, String>> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities);
+ if (!MapUtils.isEmpty(identityConfigurations)) {
+ if (propertiesToIgnore == null) {
+ propertiesToIgnore = new HashMap<>();
+ }
+
+ for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
+ String configType = entry.getKey();
+ Map<String, String> properties = entry.getValue();
+
+ if (MapUtils.isEmpty(properties)) {
+ Set<String> propertyNames = propertiesToIgnore.get(configType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<>();
+ propertiesToIgnore.put(configType, propertyNames);
+ }
+ propertyNames.addAll(properties.keySet());
+ }
+ }
+ }
+
+ return propertiesToIgnore;
+ }
+
+ /**
+ * Processes configuration changes to determine if any work needs to be done.
+ * <p/>
+ * If work is to be done, a data file containing the details is created so it they changes may be
+ * processed in the appropriate stage.
+ *
+ * @param cluster the cluster
+ * @param targetStackId the target stack id
+ * @param kerberosConfigurations the Kerberos-specific configuration map
+ * @param propertiesToBeRemoved a map of properties to be removed from the current configuration,
+ * grouped by configuration type.
+ * @param variableReplaments replacement values to use when attempting to perform variable replacements on the property names
+ * @throws AmbariException if an issue is encountered
+ */
+ private void processConfigurationChanges(Cluster cluster, StackId targetStackId,
+ KerberosDescriptor kerberosDescriptor,
+ Map<String, Map<String, String>> kerberosConfigurations,
+ Map<String, Set<String>> propertiesToBeRemoved,
+ Map<String, Map<String, String>> variableReplaments)
+ throws AmbariException {
+ actionLog.writeStdOut("Determining configuration changes");
+
+ if (!kerberosConfigurations.isEmpty()) {
+ Map<String, Service> installedServices = cluster.getServices();
+
+ // Build a map of configuration types to properties that indicate which properties should be altered
+ // This map should contain only properties defined in service-level Kerberos descriptors that
+ // have been flagged to be preconfigured and that have not yet been installed.
+ Map<String, Set<String>> propertyFilter = new HashMap<>();
+ Map<String, KerberosServiceDescriptor> serviceDescriptors = kerberosDescriptor.getServices();
+ if (serviceDescriptors != null) {
+ for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) {
+ if (!installedServices.containsKey(serviceDescriptor.getName()) && serviceDescriptor.shouldPreconfigure()) {
+ buildFilter(Collections.singleton(serviceDescriptor), propertyFilter, variableReplaments);
+ }
+ }
+ }
+
+ // Add the auth-to-local rule configuration specifications to the filter
+ Map<String, Set<String>> authToLocalProperties = kerberosHelper.translateConfigurationSpecifications(kerberosDescriptor.getAllAuthToLocalProperties());
+ if (!MapUtils.isEmpty(authToLocalProperties)) {
+ for (Map.Entry<String, Set<String>> entry : authToLocalProperties.entrySet()) {
+ Set<String> properties = entry.getValue();
+
+ if (!CollectionUtils.isEmpty(properties)) {
+ String configurationType = entry.getKey();
+
+ Set<String> propertyNames = propertyFilter.get(configurationType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<>();
+ propertyFilter.put(configurationType, propertyNames);
+ }
+
+ propertyNames.addAll(properties);
+ }
+ }
+ }
+
+ Set<String> visitedTypes = new HashSet<>();
+
+ for (Map.Entry<String, Map<String, String>> entry : kerberosConfigurations.entrySet()) {
+ String configType = entry.getKey();
+
+ String service = cluster.getServiceByConfigType(configType);
+ Set<String> allowedProperties = propertyFilter.get(configType);
+
+ // Update properties for services that are installed and not filtered out
+ if (installedServices.containsKey(service) && !CollectionUtils.isEmpty(allowedProperties)) {
+ Map<String, String> propertiesToUpdate = entry.getValue();
+ Set<String> propertiesToRemove = (propertiesToBeRemoved == null) ? null : propertiesToBeRemoved.get(configType);
+
+ // Filter the properties to update
+ if (propertiesToUpdate != null) {
+ Iterator<Map.Entry<String, String>> mapIterator = propertiesToUpdate.entrySet().iterator();
+ while (mapIterator.hasNext()) {
+ Map.Entry<String, String> mapEntry = mapIterator.next();
+
+ if (!allowedProperties.contains(mapEntry.getKey())) {
+ mapIterator.remove();
+ }
+ }
+ }
+
+ // Filter the properties to remove
+ if (propertiesToRemove != null) {
+ Iterator<String> setIterator = propertiesToRemove.iterator();
+ while (setIterator.hasNext()) {
+ String setEntry = setIterator.next();
+ if (!allowedProperties.contains(setEntry)) {
+ setIterator.remove();
+ }
+ }
+ }
+
+ visitedTypes.add(configType);
+
+ if (!MapUtils.isEmpty(propertiesToUpdate) || !CollectionUtils.isEmpty(propertiesToRemove)) {
+ if (!MapUtils.isEmpty(propertiesToUpdate)) {
+ for (Map.Entry<String, String> property : propertiesToUpdate.entrySet()) {
+ actionLog.writeStdOut(String.format("Setting: %s/%s = %s", configType, property.getKey(), property.getValue()));
+ }
+ }
+
+ if (!CollectionUtils.isEmpty(propertiesToRemove)) {
+ for (String property : propertiesToRemove) {
+ actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property));
+ }
+ }
+
+ configHelper.updateConfigType(cluster, targetStackId,
+ ambariManagementController, configType, propertiesToUpdate, propertiesToRemove,
+ ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade");
+ }
+ }
+ }
+
+ if (!MapUtils.isEmpty(propertiesToBeRemoved)) {
+ for (Map.Entry<String, Set<String>> entry : propertiesToBeRemoved.entrySet()) {
+ String configType = entry.getKey();
+
+ if (!visitedTypes.contains(configType)) {
+ Set<String> propertiesToRemove = entry.getValue();
+
+ if (!CollectionUtils.isEmpty(propertiesToRemove)) {
+ for (String property : propertiesToRemove) {
+ actionLog.writeStdOut(String.format("Removing: %s/%s", configType, property));
+ }
+
+ configHelper.updateConfigType(cluster, targetStackId,
+ ambariManagementController, configType, null, entry.getValue(),
+ ambariManagementController.getAuthName(), "Preconfiguring for Kerberos during upgrade");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds entries to the property filter (<code>propertyFilter</code>) found in the {@link KerberosConfigurationDescriptor}s
+ * within the specified node of the Kerberos descriptor.
+ *
+ * @param containers the Kerberos descriptor containers to process
+ * @param propertyFilter the property filter map to update
+ * @param replacements replacement values to use when attempting to perform variable replacements on the property names
+ * @throws AmbariException if an issue occurs while replacing variables in the property names
+ */
+ private void buildFilter(Collection<? extends AbstractKerberosDescriptorContainer> containers,
+ Map<String, Set<String>> propertyFilter,
+ Map<String, Map<String, String>> replacements)
+ throws AmbariException {
+ if (containers != null) {
+ for (AbstractKerberosDescriptorContainer container : containers) {
+ Map<String, KerberosConfigurationDescriptor> configurationDescriptors = container.getConfigurations(false);
+
+ if (!MapUtils.isEmpty(configurationDescriptors)) {
+ for (KerberosConfigurationDescriptor configurationDescriptor : configurationDescriptors.values()) {
+ Map<String, String> properties = configurationDescriptor.getProperties();
+
+ if (!MapUtils.isEmpty(properties)) {
+ String configType = configurationDescriptor.getType();
+
+ Set<String> propertyNames = propertyFilter.get(configType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<>();
+ propertyFilter.put(configType, propertyNames);
+ }
+
+ // Replace variables in the property name. For example ${knox-env/knox_user}.
+ for (String propertyName : properties.keySet()) {
+ propertyNames.add(variableReplacementHelper.replaceVariables(propertyName, replacements));
+ }
+ }
+ }
+ }
+
+ Collection<? extends AbstractKerberosDescriptorContainer> childContainers = container.getChildContainers();
+ if (childContainers != null) {
+ buildFilter(childContainers, propertyFilter, replacements);
+ }
+ }
+ }
+ }
+ }
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/RepoUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
index f68570b,2992027..db9d178
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackContext.java
@@@ -25,13 -29,17 +25,11 @@@ import java.util.concurrent.ExecutorSer
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
- import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
- import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.apache.ambari.server.state.stack.LatestRepoCallable;
import org.apache.ambari.server.state.stack.OsFamily;
-import org.apache.ambari.server.state.stack.RepoUrlInfoCallable;
-import org.apache.ambari.server.state.stack.RepoUrlInfoCallable.RepoUrlInfoResult;
-import org.apache.ambari.server.state.stack.RepoVdfCallable;
-import org.apache.commons.collections.MapUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Provides external functionality to the Stack framework.
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index d3ad351,6dc2b93..3688727
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@@ -1236,6 -1276,9 +1232,9 @@@ public class StackModule extends BaseMo
RepositoryXml serviceRepoXml = ssd.getRepoFile();
if (null != serviceRepoXml) {
repos.addAll(serviceRepoXml.getRepositories());
+ if (null != serviceRepoXml.getLatestURI()) {
- registerRepoUpdateTask(serviceRepoXml);
++ stackContext.registerRepoUpdateTask(serviceRepoXml.getLatestURI(), this);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
index 3e04a87,90dd611..23f4078
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
@@@ -26,12 -26,8 +26,9 @@@ import java.util.Set
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ClusterResponse;
import org.apache.ambari.server.controller.ServiceConfigVersionResponse;
+import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
import org.apache.ambari.server.events.ClusterConfigChangedEvent;
import org.apache.ambari.server.metadata.RoleCommandOrder;
- import org.apache.ambari.server.orm.entities.ClusterVersionEntity;
- import org.apache.ambari.server.orm.entities.HostEntity;
- import org.apache.ambari.server.orm.entities.HostVersionEntity;
import org.apache.ambari.server.orm.entities.PrivilegeEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Clusters.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
index 31a00ca,8ab1fe9..d57b5d6
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
@@@ -29,9 -29,10 +29,8 @@@ public class RepositoryInfo
private String osType;
private String repoId;
private String repoName;
- private String distribution;
- private String components;
private String mirrorsList;
private String defaultBaseUrl;
- private String latestBaseUrl;
private boolean repoSaved = false;
private boolean unique = false;
private boolean ambariManagedRepositories = true;
@@@ -186,21 -191,23 +171,20 @@@
Objects.equal(osType, that.osType) &&
Objects.equal(repoId, that.repoId) &&
Objects.equal(repoName, that.repoName) &&
- Objects.equal(distribution, that.distribution) &&
- Objects.equal(components, that.components) &&
Objects.equal(mirrorsList, that.mirrorsList) &&
Objects.equal(defaultBaseUrl, that.defaultBaseUrl) &&
- Objects.equal(latestBaseUrl, that.latestBaseUrl) &&
Objects.equal(ambariManagedRepositories, that.ambariManagedRepositories);
}
@Override
public int hashCode() {
- return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, latestBaseUrl, repoSaved, unique, ambariManagedRepositories);
- return Objects.hashCode(baseUrl, osType, repoId, repoName, distribution, components, mirrorsList, defaultBaseUrl,
- ambariManagedRepositories);
++ return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, repoSaved, unique, ambariManagedRepositories);
}
public RepositoryResponse convertToResponse()
{
return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(),
- getRepoName(), getMirrorsList(), getDefaultBaseUrl(), getLatestBaseUrl());
- getRepoName(), getDistribution(), getComponents(), getMirrorsList(), getDefaultBaseUrl());
++ getRepoName(), getMirrorsList(), getDefaultBaseUrl());
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
index 91884f6,65189ca..b6203f9
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Service.java
@@@ -23,7 -22,7 +23,8 @@@ import java.util.Set
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ServiceResponse;
+import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
public interface Service {
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
index d755568,9fb2aba..ed19023
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponent.java
@@@ -23,7 -22,7 +23,8 @@@ import java.util.Set
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ServiceComponentResponse;
+import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
+ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
public interface ServiceComponent {
@@@ -102,5 -102,20 +108,20 @@@
ServiceComponentHost addServiceComponentHost(
String hostName) throws AmbariException;
- void delete() throws AmbariException;
+ void delete(DeleteHostComponentStatusMetaData deleteMetaData);
+
+ /**
+ * This method computes the state of the repository that's associated with the desired
+ * version. It is used, for example, when a host component reports its version and the
+ * state can be in flux.
+ *
+ * @param reportedVersion
+ * @throws AmbariException
+ */
+ void updateRepositoryState(String reportedVersion) throws AmbariException;
+
+ /**
+ * @return the repository state for the desired version
+ */
+ RepositoryVersionState getRepositoryState();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
index 08f9f41,5ff9e37..575193f
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
@@@ -23,9 -23,8 +23,9 @@@ import java.util.Map
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ServiceComponentHostResponse;
+import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
- import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
+ import org.apache.ambari.server.orm.entities.HostVersionEntity;
import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
@@@ -130,30 -107,9 +108,9 @@@ public interface ServiceComponentHost
*
* @param version component version (e.g. 2.2.0.0-2041)
*/
- void setVersion(String version);
+ void setVersion(String version) throws AmbariException;
/**
- * Gets the desired security state for this ServiceComponent
- * <p/>
- * The returned SecurityState is a valid endpoint state where
- * SecurityState.isEndpoint() == true.
- *
- * @return the desired SecurityState for this ServiceComponent
- */
- SecurityState getDesiredSecurityState();
-
- /**
- * Sets the desired security state for this ServiceComponent
- * <p/>
- * It is expected that the new SecurityState is a valid endpoint state such that
- * SecurityState.isEndpoint() == true.
- *
- * @param securityState the desired SecurityState for this ServiceComponent
- * @throws AmbariException if the new state is not an endpoint state
- */
- void setDesiredSecurityState(SecurityState securityState) throws AmbariException;
-
- /**
* @param upgradeState the upgrade state
*/
void setUpgradeState(UpgradeState upgradeState);
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
index 32ecbf8,22c97ed..74b592b
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
@@@ -19,10 -19,9 +19,11 @@@
package org.apache.ambari.server.state;
import java.util.HashMap;
+import java.util.HashSet;
+ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReadWriteLock;
@@@ -32,9 -31,10 +33,10 @@@ import org.apache.ambari.server.AmbariE
import org.apache.ambari.server.ObjectNotFoundException;
import org.apache.ambari.server.ServiceComponentHostNotFoundException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
-import org.apache.ambari.server.controller.MaintenanceStateHelper;
import org.apache.ambari.server.controller.ServiceComponentResponse;
+import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
import org.apache.ambari.server.events.ServiceComponentRecoveryChangedEvent;
+ import org.apache.ambari.server.events.listeners.upgrade.StackVersionListener;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.orm.dao.ClusterServiceDAO;
import org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO;
@@@ -84,11 -92,15 +94,12 @@@ public class ServiceComponentImpl imple
*/
private final long desiredStateEntityId;
- /**
- * Data access object used for lookup up stacks.
- */
- private final StackDAO stackDAO;
+ @Inject
+ private RepositoryVersionDAO repoVersionDAO;
+
+ @Inject
+ private HostComponentStateDAO hostComponentDAO;
- @Inject
- private MaintenanceStateHelper maintenanceStateHelper;
-
@AssistedInject
public ServiceComponentImpl(@Assisted Service service, @Assisted String componentName,
AmbariMetaInfo ambariMetaInfo,
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index c22bfb2,1104d19..8ba41ae
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@@ -18,14 -18,13 +18,16 @@@
package org.apache.ambari.server.state;
+ import java.util.ArrayList;
+ import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@@ -588,25 -577,22 +595,30 @@@ public class ServiceImpl implements Ser
@Override
@Transactional
- public void delete() throws AmbariException {
+ public void delete(DeleteHostComponentStatusMetaData deleteMetaData) {
+ List<Component> components = getComponents(); // XXX temporal coupling, need to call this BEFORE deletingAllComponents
- deleteAllComponents();
- deleteAllServiceConfigs();
+ deleteAllComponents(deleteMetaData);
+ if (deleteMetaData.getAmbariException() != null) {
+ return;
+ }
+ try {
+ deleteAllServiceConfigs();
- StackId stackId = getDesiredStackId();
+ removeEntities();
+ } catch (AmbariException e) {
+ deleteMetaData.setAmbariException(e);
+ return;
+ }
- removeEntities();
++ StackId stackId = getDesiredStackId();
+
// publish the service removed event
- StackId stackId = cluster.getDesiredStackVersion();
+ if (null == stackId) {
+ return;
+ }
ServiceRemovedEvent event = new ServiceRemovedEvent(getClusterId(), stackId.getStackName(),
- stackId.getStackVersion(), getName());
+ stackId.getStackVersion(), getName(), components);
eventPublisher.publish(event);
}
@@@ -635,16 -626,9 +652,12 @@@
@Override
public MaintenanceState getMaintenanceState() {
- return getServiceDesiredStateEntity().getMaintenanceState();
+ if (maintenanceState.get() == null) {
+ maintenanceState.set(getServiceDesiredStateEntity().getMaintenanceState());
+ }
+ return maintenanceState.get();
}
- private ClusterServiceEntity getServiceEntity() {
- return clusterServiceDAO.findByPK(serviceEntityPK);
- }
-
private ClusterServiceEntityPK getServiceEntityPK(ClusterServiceEntity serviceEntity) {
ClusterServiceEntityPK pk = new ClusterServiceEntityPK();
pk.setClusterId(serviceEntity.getClusterId());
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index 353dd86,a3886ab..6184b94
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@@ -33,6 -33,6 +33,7 @@@ import org.apache.ambari.server.control
import org.apache.ambari.server.stack.Validable;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
++import org.apache.ambari.server.state.stack.LatestRepoCallable;
import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
import org.apache.ambari.server.state.stack.UpgradePack;
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 92e01c2,8f9d8e1..464cb41
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@@ -491,6 -553,51 +553,36 @@@ public class UpgradeHelper
}
/**
+ * Merges two service check groups when they have been orchestrated back-to-back.
+ * @param newHolder the "new" group holder, which was orchestrated after the "old" one
+ * @param oldHolder the "old" group holder, which is one that was already orchestrated
+ */
+ @SuppressWarnings("unchecked")
+ private void mergeServiceChecks(UpgradeGroupHolder newHolder, UpgradeGroupHolder oldHolder) {
+
+ LinkedHashSet<StageWrapper> priority = new LinkedHashSet<>();
+ LinkedHashSet<StageWrapper> others = new LinkedHashSet<>();
+
- Set<String> extraKeys = new HashSet<>();
- LinkedHashSet<StageWrapper> extras = new LinkedHashSet<>();
-
+ for (List<StageWrapper> holderItems : new List[] { oldHolder.items, newHolder.items }) {
+ for (StageWrapper stageWrapper : holderItems) {
- if (stageWrapper instanceof ServiceCheckStageWrapper) {
- ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
- if (wrapper.priority) {
- priority.add(stageWrapper);
- } else {
- others.add(stageWrapper);
- }
++ ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
++
++ if (wrapper.priority) {
++ priority.add(stageWrapper);
+ } else {
- // !!! It's a good chance that back-to-back service check groups are adding the
- // same non-service-check wrappers.
- // this should be "equal enough" to prevent them from duplicating on merge
- String key = stageWrapper.toString();
- if (!extraKeys.contains(key)) {
- extras.add(stageWrapper);
- extraKeys.add(key);
- }
++ others.add(stageWrapper);
+ }
-
+ }
+ }
+
+ // !!! remove duplicate wrappers that are now in the priority list
+ others = new LinkedHashSet<>(CollectionUtils.subtract(others, priority));
+
+ oldHolder.items = Lists.newLinkedList(priority);
+ oldHolder.items.addAll(others);
- oldHolder.items.addAll(extras);
+ }
+
+ /**
* Walks through the UpgradeGroupHolder and updates titles and manual tasks,
* replacing keyword tokens needed for display purposes
*
@@@ -768,8 -905,250 +890,241 @@@
serviceComponentHost.setVersion(StackVersionListener.UNKNOWN_VERSION);
}
}
- serviceComponent.setDesiredVersion(desiredVersion);
+ // set component desired repo
+ serviceComponent.setDesiredRepositoryVersion(targetRepositoryVersion);
+ }
+ }
+ }
+
+ /**
+ * Handles the creation or resetting of configurations based on whether an
+ * upgrade or downgrade is occurring. This method will not do anything when
+ * the service is not crossing major stack versions, since, by definition, no
+ * new configurations are automatically created when upgrading with the same
+ * stack (ie HDP 2.2.0.0 -> HDP 2.2.1.0).
+ * <p/>
+ * When upgrading or downgrade between stacks (HDP 2.2.0.0 -> HDP 2.3.0.0)
+ * then this will perform the following:
+ * <ul>
+ * <li>Upgrade: Create new configurations that are a merge between the source
+ * stack and the target stack. If a value has changed between stacks, then the
+ * target stack value should be taken unless the cluster's value differs from
+ * the old stack. This can occur if a property has been customized after
- * installation. Read-only properties, however, are always taken from the new
- * stack.</li>
++ * installation.</li>
+ * <li>Downgrade: Reset the latest configurations from the service's original
+ * stack. The new configurations that were created on upgrade must be left
+ * intact until all components have been reverted, otherwise heartbeats will
+ * fail due to missing configurations.</li>
+ * </ul>
+ *
+ * @param upgradeContext
+ * the upgrade context (not {@code null}).
+ * @throws AmbariException
+ */
+ private void processConfigurationsIfRequired(UpgradeContext upgradeContext)
+ throws AmbariException {
+
+ AmbariManagementController controller = m_controllerProvider.get();
+
+ Cluster cluster = upgradeContext.getCluster();
+ Direction direction = upgradeContext.getDirection();
+ String userName = controller.getAuthName();
+ Set<String> servicesInUpgrade = upgradeContext.getSupportedServices();
+
+ Set<String> clusterConfigTypes = new HashSet<>();
+ Set<String> processedClusterConfigTypes = new HashSet<>();
+
+ // merge or revert configurations for any service that needs it
+ for (String serviceName : servicesInUpgrade) {
+ RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName);
+ RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName);
+ StackId sourceStackId = sourceRepositoryVersion.getStackId();
+ StackId targetStackId = targetRepositoryVersion.getStackId();
+
+ // only work with configurations when crossing stacks
+ if (sourceStackId.equals(targetStackId)) {
+ RepositoryVersionEntity associatedRepositoryVersion = upgradeContext.getRepositoryVersion();
+ LOG.info(
+ "The {} {} {} will not change stack configurations for {} since the source and target are both {}",
+ direction.getText(false), direction.getPreposition(),
+ associatedRepositoryVersion.getVersion(), serviceName, targetStackId);
+
+ continue;
+ }
+
+ ConfigHelper configHelper = m_configHelperProvider.get();
+
+ // downgrade is easy - just remove the new and make the old current
+ if (direction == Direction.DOWNGRADE) {
+ cluster.applyLatestConfigurations(targetStackId, serviceName);
+ continue;
+ }
+
- // the auto-merge must take read-only properties even if they have changed
- // - if the properties was read-only in the source stack, then we must
- // take the new stack's value
- Map<String, Set<String>> readOnlyProperties = getReadOnlyProperties(sourceStackId, serviceName);
-
+ // upgrade is a bit harder - we have to merge new stack configurations in
+
+ // populate a map of default configurations for the service on the old
+ // stack (this is used when determining if a property has been
+ // customized and should be overriden with the new stack value)
+ Map<String, Map<String, String>> oldServiceDefaultConfigsByType = configHelper.getDefaultProperties(
+ sourceStackId, serviceName);
+
+ // populate a map with default configurations from the new stack
+ Map<String, Map<String, String>> newServiceDefaultConfigsByType = configHelper.getDefaultProperties(
+ targetStackId, serviceName);
+
+ if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) {
+ continue;
+ }
+
+ Set<String> foundConfigTypes = new HashSet<>();
+
+ // find the current, existing configurations for the service
+ List<Config> existingServiceConfigs = new ArrayList<>();
+ List<ServiceConfigEntity> latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService(
+ cluster.getClusterId(), serviceName);
+
+ for (ServiceConfigEntity serviceConfig : latestServiceConfigs) {
+ List<ClusterConfigEntity> existingConfigurations = serviceConfig.getClusterConfigEntities();
+ for (ClusterConfigEntity currentServiceConfig : existingConfigurations) {
+ String configurationType = currentServiceConfig.getType();
+
+ Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType);
+ existingServiceConfigs.add(currentClusterConfigForService);
+ foundConfigTypes.add(configurationType);
+ }
+ }
+
+ // !!! these are the types that come back from the config helper, but are not part of the service.
+ @SuppressWarnings("unchecked")
+ Set<String> missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(),
+ foundConfigTypes));
+
+ for (String missingConfigType : missingConfigTypes) {
+ Config config = cluster.getDesiredConfigByType(missingConfigType);
+ if (null != config) {
+ existingServiceConfigs.add(config);
+ clusterConfigTypes.add(missingConfigType);
+ }
+ }
+
+ // now that we have found, old, new, and existing confgs, overlay the
+ // existing on top of the new
+ for (Config existingServiceConfig : existingServiceConfigs) {
+ String configurationType = existingServiceConfig.getType();
+
+ // get current stack default configurations on install
+ Map<String, String> oldServiceDefaultConfigs = oldServiceDefaultConfigsByType.get(
+ configurationType);
+
+ // NPE sanity for current stack defaults
+ if (null == oldServiceDefaultConfigs) {
+ oldServiceDefaultConfigs = Collections.emptyMap();
+ }
+
+ // get the existing configurations
+ Map<String, String> existingConfigurations = existingServiceConfig.getProperties();
+
+ // get the new configurations
- Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(configurationType);
++ Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(
++ configurationType);
+
+ // if the new stack configurations don't have the type, then simply add
+ // all of the existing in
+ if (null == newDefaultConfigurations) {
+ newServiceDefaultConfigsByType.put(configurationType, existingConfigurations);
+ continue;
+ } else {
+ // Remove any configs in the new stack whose value is NULL, unless
+ // they currently exist and the value is not NULL.
+ Iterator<Map.Entry<String, String>> iter = newDefaultConfigurations.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ if (entry.getValue() == null) {
+ iter.remove();
+ }
+ }
+ }
+
- // process every existing configuration property for this configuration type
++ // process every existing configuration property for this configuration
++ // type
+ for (Map.Entry<String, String> existingConfigurationEntry : existingConfigurations.entrySet()) {
+ String existingConfigurationKey = existingConfigurationEntry.getKey();
+ String existingConfigurationValue = existingConfigurationEntry.getValue();
+
+ // if there is already an entry, we now have to try to determine if
+ // the value was customized after stack installation
+ if (newDefaultConfigurations.containsKey(existingConfigurationKey)) {
+ String newDefaultConfigurationValue = newDefaultConfigurations.get(
+ existingConfigurationKey);
+
+ if (!StringUtils.equals(existingConfigurationValue, newDefaultConfigurationValue)) {
+ // the new default is different from the existing cluster value;
+ // only override the default value if the existing value differs
+ // from the original stack
+ String oldDefaultValue = oldServiceDefaultConfigs.get(existingConfigurationKey);
+
- // see if this property is a read-only property which means that
- // we shouldn't care if it was changed - we should take the new
- // stack's value
- Set<String> readOnlyPropertiesForType = readOnlyProperties.get(configurationType);
- boolean readOnly = (null != readOnlyPropertiesForType
- && readOnlyPropertiesForType.contains(existingConfigurationKey));
-
- if (!readOnly && !StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
- // at this point, we've determined that there is a difference
- // between default values between stacks, but the value was also
- // customized, so keep the customized value
++ if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
++ // at this point, we've determined that there is a
++ // difference
++ // between default values between stacks, but the value was
++ // also customized, so keep the customized value
+ newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
+ }
+ }
+ } else {
- // there is no entry in the map, so add the existing key/value pair
++ // there is no entry in the map, so add the existing key/value
++ // pair
+ newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
+ }
+ }
+
+ /*
+ for every new configuration which does not exist in the existing
+ configurations, see if it was present in the current stack
+
+ stack 2.x has foo-site/property (on-ambari-upgrade is false)
+ stack 2.y has foo-site/property
+ the current cluster (on 2.x) does not have it
+
+ In this case, we should NOT add it back as clearly stack advisor has removed it
+ */
+ Iterator<Map.Entry<String, String>> newDefaultConfigurationsIterator = newDefaultConfigurations.entrySet().iterator();
+ while (newDefaultConfigurationsIterator.hasNext()) {
+ Map.Entry<String, String> newConfigurationEntry = newDefaultConfigurationsIterator.next();
+ String newConfigurationPropertyName = newConfigurationEntry.getKey();
+ if (oldServiceDefaultConfigs.containsKey(newConfigurationPropertyName)
+ && !existingConfigurations.containsKey(newConfigurationPropertyName)) {
+ LOG.info(
+ "The property {}/{} exists in both {} and {} but is not part of the current set of configurations and will therefore not be included in the configuration merge",
+ configurationType, newConfigurationPropertyName, sourceStackId, targetStackId);
+
+ // remove the property so it doesn't get merged in
+ newDefaultConfigurationsIterator.remove();
+ }
+ }
+ }
+
+ if (null != newServiceDefaultConfigsByType) {
+
+ for (String clusterConfigType : clusterConfigTypes) {
+ if (processedClusterConfigTypes.contains(clusterConfigType)) {
+ newServiceDefaultConfigsByType.remove(clusterConfigType);
+ } else {
+ processedClusterConfigTypes.add(clusterConfigType);
+ }
+
+ }
+
+ Set<String> configTypes = newServiceDefaultConfigsByType.keySet();
+ LOG.warn("The upgrade will create the following configurations for stack {}: {}",
+ targetStackId, StringUtils.join(configTypes, ','));
+
+ String serviceVersionNote = String.format("%s %s %s", direction.getText(true),
+ direction.getPreposition(), upgradeContext.getRepositoryVersion().getVersion());
+
+ configHelper.createConfigTypes(cluster, targetStackId, controller,
+ newServiceDefaultConfigsByType, userName, serviceVersionNote);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertDefinitionHash.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/ParameterizedSource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Reporting.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/alert/Source.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 7147ce3,9c0b0ca..e6da65e
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@@ -145,14 -128,12 +132,16 @@@ import org.apache.ambari.server.state.U
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.state.configgroup.ConfigGroupFactory;
import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
+ import org.apache.ambari.server.state.repository.ClusterVersionSummary;
+ import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.scheduler.RequestExecution;
import org.apache.ambari.server.state.scheduler.RequestExecutionFactory;
+import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.svccomphost.ServiceComponentHostSummary;
+import org.apache.ambari.server.topology.TopologyDeleteFormer;
import org.apache.ambari.server.topology.TopologyRequest;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -2315,15 -1545,10 +1569,15 @@@ public class ClusterImpl implements Clu
serviceConfigEntity.setVersion(nextServiceConfigVersion);
serviceConfigEntity.setUser(user);
serviceConfigEntity.setNote(note);
- serviceConfigEntity.setStack(clusterEntity.getDesiredStack());
+ serviceConfigEntity.setStack(stackEntity);
serviceConfigDAO.create(serviceConfigEntity);
+ List<String> groupHostNames = null;
if (configGroup != null) {
+ if (MapUtils.isNotEmpty(configGroup.getHosts())) {
+ groupHostNames = configGroup.getHosts().entrySet().stream().map(h -> h.getValue().getHostName())
+ .collect(Collectors.toList());
+ }
serviceConfigEntity.setHostIds(new ArrayList<>(configGroup.getHosts().keySet()));
serviceConfigEntity = serviceConfigDAO.merge(serviceConfigEntity);
}
@@@ -2585,19 -1806,31 +1837,32 @@@
throw new ObjectNotFoundException("Service config version with serviceName={} and version={} not found");
}
+ String configGroupName = null;
// disable all configs related to service
if (serviceConfigEntity.getGroupId() == null) {
+ // Here was fixed bug with entity changes revert. More you can find here AMBARI-21173.
+ // This issue reproduces only if you are changing same entity in first and second loop.
+ // In that case eclipselink will revert changes to cached, if entity has fluchGroup and it
+ // needs to be refreshed. Actually we don't need to change same antities in few steps, so i
+ // decided to filter out. duplicates and do not change them. It will be better for performance and bug will be fixed.
Collection<String> configTypes = serviceConfigTypes.get(serviceName);
List<ClusterConfigEntity> enabledConfigs = clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
+ List<ClusterConfigEntity> serviceConfigEntities = serviceConfigEntity.getClusterConfigEntities();
+ ArrayList<ClusterConfigEntity> duplicatevalues = new ArrayList<>(serviceConfigEntities);
+ duplicatevalues.retainAll(enabledConfigs);
+
for (ClusterConfigEntity enabledConfig : enabledConfigs) {
- enabledConfig.setSelected(false);
- clusterDAO.merge(enabledConfig);
+ if (!duplicatevalues.contains(enabledConfig)) {
+ enabledConfig.setSelected(false);
+ clusterDAO.merge(enabledConfig);
+ }
}
- for (ClusterConfigEntity configEntity : serviceConfigEntity.getClusterConfigEntities()) {
- configEntity.setSelected(true);
- clusterDAO.merge(configEntity);
+ for (ClusterConfigEntity configEntity : serviceConfigEntities) {
+ if (!duplicatevalues.contains(configEntity)) {
+ configEntity.setSelected(true);
+ clusterDAO.merge(configEntity);
+ }
}
} else {
Long configGroupId = serviceConfigEntity.getGroupId();
@@@ -3159,9 -2415,14 +2463,14 @@@
// since the entities which were modified came from the cluster entity's
// list to begin with, we can just save them right back - no need for a
// new collection since the entity instances were modified directly
- clusterEntity = clusterDAO.merge(clusterEntity, true);
+ clusterEntity = clusterDAO.merge(clusterEntity);
cacheConfigurations();
+
+ LOG.info(
+ "Applied latest configurations for {} on stack {}. The the following types were modified: {}",
+ serviceName, stackId, StringUtils.join(configTypesForService, ','));
+
} finally {
clusterGlobalLock.writeLock().unlock();
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
index 702d776,5ac1ac3..8743f63
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClustersImpl.java
@@@ -46,13 -40,10 +46,11 @@@ import org.apache.ambari.server.control
import org.apache.ambari.server.events.HostRegisteredEvent;
import org.apache.ambari.server.events.HostsAddedEvent;
import org.apache.ambari.server.events.HostsRemovedEvent;
+import org.apache.ambari.server.events.TopologyUpdateEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.orm.dao.ClusterDAO;
- import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
import org.apache.ambari.server.orm.dao.HostConfigMappingDAO;
import org.apache.ambari.server.orm.dao.HostDAO;
- import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.orm.dao.HostStateDAO;
import org.apache.ambari.server.orm.dao.HostVersionDAO;
import org.apache.ambari.server.orm.dao.KerberosPrincipalHostDAO;
@@@ -276,17 -243,9 +260,17 @@@ public class ClustersImpl implements Cl
clusters.put(clusterName, cluster);
clustersById.put(cluster.getClusterId(), cluster);
clusterHostMap.put(clusterName,
- Collections.newSetFromMap(new ConcurrentHashMap<Host, Boolean>()));
+ Collections.newSetFromMap(new ConcurrentHashMap<>()));
cluster.setCurrentStackVersion(stackId);
+
+ TreeMap<String, TopologyCluster> addedClusters = new TreeMap<>();
+ TopologyCluster addedCluster = new TopologyCluster();
+ addedClusters.put(Long.toString(cluster.getClusterId()), addedCluster);
+ TopologyUpdateEvent topologyUpdateEvent = new TopologyUpdateEvent(addedClusters,
+ TopologyUpdateEvent.EventType.UPDATE);
+ m_topologyHolder.get().updateData(topologyUpdateEvent);
+ m_metadataHolder.get().updateData(m_ambariManagementController.get().getClusterMetadata(cluster));
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 0b54eec,274a425..2a9e69e
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@@ -595,14 -580,9 +586,10 @@@ public class HostImpl implements Host
writeLock.unlock();
}
if (oldState != getState()) {
+ ambariEventPublisher.publish(new HostStateUpdateEvent(getHostName(), getState()));
if (LOG.isDebugEnabled()) {
- LOG.debug("Host transitioned to a new state"
- + ", host=" + getHostName()
- + ", oldState=" + oldState
- + ", currentState=" + getState()
- + ", eventType=" + event.getType().name()
- + ", event=" + event);
+ LOG.debug("Host transitioned to a new state, host={}, oldState={}, currentState={}, eventType={}, event={}",
+ getHostName(), oldState, getState(), event.getType().name(), event);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
index a1b9e5c,0c7a9a9..9432f6c
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
@@@ -193,13 -201,17 +198,17 @@@ public class KerberosDescriptor extend
}
if (services == null) {
- services = new TreeMap<String, KerberosServiceDescriptor>();
+ services = new TreeMap<>();
}
- services.put(name, service);
-
- // Set the service's parent to this KerberosDescriptor
- service.setParent(this);
+ KerberosServiceDescriptor existing = services.get(name);
+ if (existing == null) {
+ services.put(name, service);
+ // Set the service's parent to this KerberosDescriptor
+ service.setParent(this);
+ } else {
+ existing.update(service);
+ }
}
}
@@@ -418,4 -425,43 +422,43 @@@
return authToLocalProperties;
}
+
+ /**
+ * Get a map of principals, where the key is the principal path (SERVICE/COMPONENT/principal_name or SERVICE/principal_name) and the value is the principal.
+ * <p>
+ * For example if the kerberos principal of the HISTORYSERVER is defined in the kerberos.json:
+ * "name": "history_server_jhs",
+ * "principal": {
+ * "value": "jhs/_HOST@${realm}",
+ * "type" : "service",
+ * },
+ * Then "jhs/_HOST@EXAMPLE.COM" will be put into the map under the "MAPREDUCE2/HISTORYSERVER/history_server_jhs" key.
+ */
+ public Map<String, String> principals() throws AmbariException {
+ Map<String, String> result = new HashMap<>();
+ for (AbstractKerberosDescriptorContainer each : nullToEmpty(getChildContainers())) {
+ if ((each instanceof KerberosServiceDescriptor)) {
+ collectFromComponents(each.getName(), nullToEmpty(((KerberosServiceDescriptor) each).getComponents()).values(), result);
+ collectFromIdentities(each.getName(), "", nullToEmpty(each.getIdentities()), result);
+ }
+ }
+ return result;
+ }
+
+ private static void collectFromComponents(String service, Collection<KerberosComponentDescriptor> components, Map<String, String> result) {
+ for (KerberosComponentDescriptor each : components) {
+ collectFromIdentities(service, each.getName(), nullToEmpty(each.getIdentities()), result);
+ }
+ }
+
+ private static void collectFromIdentities(String service, String component, Collection<KerberosIdentityDescriptor> identities, Map<String, String> result) {
+ for (KerberosIdentityDescriptor each : identities) {
- if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent()) {
++ if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent() && !each.getName().startsWith("/")) {
+ String path = StringUtils.isBlank(component)
+ ? String.format("%s/%s", service, each.getName())
+ : String.format("%s/%s/%s", service, component, each.getName());
+ result.put(path, each.getPrincipalDescriptor().getName());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
index 68cafe5,51b7cd0..5da3399
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
@@@ -157,6 -150,8 +146,8 @@@ public class KerberosServiceDescriptor
}
}
}
+
- setPreconfigure(getBooleanValue(data, KEY_PRECONFIGURE));
++ setPreconfigure(getBooleanValue(data, "preconfigure"));
}
}
@@@ -266,9 -279,13 +275,13 @@@
for (KerberosComponentDescriptor component : components.values()) {
list.add(component.toMap());
}
- map.put(KEY_COMPONENTS, list);
+ map.put(Type.COMPONENT.getDescriptorPluralName(), list);
}
+ if (preconfigure != null) {
- map.put(KEY_PRECONFIGURE, preconfigure.toString());
++ map.put("preProcess", preconfigure.toString());
+ }
+
return map;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
index 078b4ae,c2209bb..03b3705
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
@@@ -146,7 -146,8 +146,6 @@@ public class RepositoryXml implements V
private String mirrorslist = null;
private String repoid = null;
private String reponame = null;
- private String latest = null;
- private String distribution = null;
- private String components = null;
private boolean unique = false;
private Repo() {
@@@ -180,10 -181,13 +179,6 @@@
return reponame;
}
- public String getLatestUri() {
- return latest;
- public String getDistribution() {
- return distribution;
-- }
--
- public String getComponents() {
- return components;
- }
/**
* @return true if version of HDP that change with each release
*/
@@@ -217,7 -221,8 +212,6 @@@
ri.setOsType(os.trim());
ri.setRepoId(r.getRepoId());
ri.setRepoName(r.getRepoName());
- ri.setLatestBaseUrl(r.getBaseUrl());
- ri.setDistribution(r.getDistribution());
- ri.setComponents(r.getComponents());
ri.setUnique(r.isUnique());
repos.add(ri);
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index fdb7c8d,f540d8d..9524c09
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@@ -218,4 -246,98 +236,97 @@@ public class RepositoryVersionHelper
throw new AmbariException("There were no suitable upgrade packs for stack " + stackName + " " + stackVersion +
((null != upgradeType) ? " and upgrade type " + upgradeType : ""));
}
+
+ /**
+ * Build the role parameters for an install command.
+ *
+ * @param amc the management controller. Tests don't use the same instance that gets injected.
+ * @param repoVersion the repository version
+ * @param osFamily the os family
+ * @param servicesOnHost the set of services to check for packages
+ * @return a Map<String, String> to use in
+ */
+ public Map<String, String> buildRoleParams(AmbariManagementController amc, RepositoryVersionEntity repoVersion, String osFamily, Set<String> servicesOnHost)
+ throws SystemException {
+
+ StackId stackId = repoVersion.getStackId();
+
+ List<ServiceOsSpecific.Package> packages = new ArrayList<>();
+
+ for (String serviceName : servicesOnHost) {
+ ServiceInfo info;
+
+ try {
+ if (ami.get().isServiceRemovedInStack(stackId.getStackName(), stackId.getStackVersion(), serviceName)) {
+ LOG.info(String.format("%s has been removed from stack %s-%s. Skip calculating its installation packages", stackId.getStackName(), stackId.getStackVersion(), serviceName));
+ continue; //No need to calculate install packages for removed services
+ }
+
+ info = ami.get().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName);
+ } catch (AmbariException e) {
+ throw new SystemException(String.format("Cannot obtain stack information for %s-%s", stackId.getStackName(), stackId.getStackVersion()), e);
+ }
+
+ List<ServiceOsSpecific.Package> packagesForService = amc.getPackagesForServiceHost(info,
+ new HashMap<>(), osFamily);
+
+ List<String> blacklistedPackagePrefixes = configuration.get().getRollingUpgradeSkipPackagesPrefixes();
+
+ for (ServiceOsSpecific.Package aPackage : packagesForService) {
+ if (!aPackage.getSkipUpgrade()) {
+ boolean blacklisted = false;
+ for (String prefix : blacklistedPackagePrefixes) {
+ if (aPackage.getName().startsWith(prefix)) {
+ blacklisted = true;
+ break;
+ }
+ }
+ if (! blacklisted) {
+ packages.add(aPackage);
+ }
+ }
+ }
+ }
+
+ Map<String, String> roleParams = new HashMap<>();
+ roleParams.put("stack_id", stackId.getStackId());
+ // !!! TODO make roleParams <String, Object> so we don't have to do this awfulness.
+ roleParams.put(KeyNames.PACKAGE_LIST, gson.toJson(packages));
+
+ return roleParams;
+ }
+
+ /**
+ * Adds a command repository to the action context
+ * @param context the context
+ * @param osFamily the OS family
+ * @param repoVersion the repository version entity
+ * @param repos the repository entities
+ */
+ public void addCommandRepository(ActionExecutionContext context,
+ RepositoryVersionEntity repoVersion, OperatingSystemEntity osEntity) {
+
+ final CommandRepository commandRepo = new CommandRepository();
+ commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories());
+ commandRepo.setRepositoryVersion(repoVersion.getVersion());
+ commandRepo.setRepositoryVersionId(repoVersion.getId());
- commandRepo.setResolved(repoVersion.isResolved());
+ commandRepo.setStackName(repoVersion.getStackId().getStackName());
+
+ if (!osEntity.isAmbariManagedRepos()) {
+ commandRepo.setNonManaged();
+ } else {
+ commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId()));
+ }
+
+ context.addVisitor(new ExecutionCommandVisitor() {
+ @Override
+ public void visit(ExecutionCommand command) {
+ if (null == command.getRepositoryFile()) {
+ command.setRepositoryFile(commandRepo);
+ }
+ }
+ });
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/be73d167/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 446ae8c,230b031..ecb616a
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@@ -25,10 -24,8 +25,11 @@@ import java.util.HashSet
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@@ -93,8 -83,8 +95,9 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
+ import com.google.common.util.concurrent.Striped;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import com.google.inject.persist.Transactional;
@@@ -880,8 -869,8 +888,9 @@@ public class ServiceComponentHostImpl i
}
@Override
+ @Transactional
public void setState(State state) {
+ State oldState = getState();
stateMachine.setCurrentState(state);
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
@@@ -913,7 -898,8 +922,8 @@@
}
@Override
+ @Transactional
- public void setVersion(String version) {
+ public void setVersion(String version) throws AmbariException {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
stateEntity.setVersion(version);