You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/07/12 21:00:54 UTC
[1/6] ambari git commit: AMBARI-21392. Cleanup relevant Kerberos
identities when a service is removed (amagyar)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-20859 709710915 -> 7a0850b49
AMBARI-21392. Cleanup relevant Kerberos identities when a service is removed (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e767aa44
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e767aa44
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e767aa44
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: e767aa44d872bab9ac0c416684f80b2b662347e5
Parents: 0b397cd
Author: Attila Magyar <am...@hortonworks.com>
Authored: Tue Jul 11 20:10:12 2017 +0200
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Tue Jul 11 20:10:12 2017 +0200
----------------------------------------------------------------------
.../controller/DeleteIdentityHandler.java | 77 ++++++++--
.../server/controller/KerberosHelper.java | 2 +-
.../server/controller/KerberosHelperImpl.java | 5 +-
.../utilities/KerberosIdentityCleaner.java | 88 +++--------
.../utilities/RemovableIdentities.java | 145 +++++++++++++++++++
.../controller/utilities/UsedIdentities.java | 101 +++++++++++++
.../ServiceComponentUninstalledEvent.java | 6 +
.../server/events/ServiceRemovedEvent.java | 29 ++--
.../ambari/server/orm/dao/ClusterDAO.java | 15 ++
.../orm/entities/ClusterConfigEntity.java | 3 +
.../org/apache/ambari/server/state/Cluster.java | 7 +
.../apache/ambari/server/state/ServiceImpl.java | 14 +-
.../server/state/cluster/ClusterImpl.java | 9 ++
.../AbstractKerberosDescriptorContainer.java | 12 ++
.../kerberos/KerberosComponentDescriptor.java | 15 --
.../kerberos/KerberosIdentityDescriptor.java | 14 +-
.../utilities/KerberosIdentityCleanerTest.java | 102 +++++++++++--
.../server/orm/dao/ServiceConfigDAOTest.java | 12 ++
18 files changed, 520 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
index aa098b6..3329e76 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/DeleteIdentityHandler.java
@@ -17,12 +17,13 @@
*/
package org.apache.ambari.server.controller;
-import static com.google.common.collect.Sets.newHashSet;
+import static java.util.Collections.singleton;
+import static java.util.stream.Collectors.toSet;
import static org.apache.ambari.server.controller.KerberosHelperImpl.BASE_LOG_DIR;
import java.io.File;
+import java.lang.reflect.Type;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -45,10 +46,15 @@ import org.apache.ambari.server.serveraction.kerberos.KDCType;
import org.apache.ambari.server.serveraction.kerberos.KerberosOperationHandler;
import org.apache.ambari.server.serveraction.kerberos.KerberosServerAction;
import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent;
import org.apache.ambari.server.utils.StageUtils;
+import com.google.gson.reflect.TypeToken;
+
+
/**
* I delete kerberos identities (principals and keytabs) of a given component.
*/
@@ -78,7 +84,7 @@ class DeleteIdentityHandler {
if (manageIdentities) {
addPrepareDeleteIdentity(cluster, hostParamsJson, event, commandParameters, stageContainer);
addDestroyPrincipals(cluster, hostParamsJson, event, commandParameters, stageContainer);
- addDeleteKeytab(cluster, newHashSet(commandParameters.component.getHostName()), hostParamsJson, commandParameters, stageContainer);
+ addDeleteKeytab(cluster, commandParameters.getAffectedHostNames(), hostParamsJson, commandParameters, stageContainer);
}
addFinalize(cluster, hostParamsJson, event, stageContainer, commandParameters);
}
@@ -172,15 +178,15 @@ class DeleteIdentityHandler {
public static class CommandParams {
- private final Component component;
- private final List<String> identities;
+ private final List<Component> components;
+ private final Set<String> identities;
private final String authName;
private final File dataDirectory;
private final String defaultRealm;
private final KDCType kdcType;
- public CommandParams(Component component, List<String> identities, String authName, File dataDirectory, String defaultRealm, KDCType kdcType) {
- this.component = component;
+ public CommandParams(List<Component> components, Set<String> identities, String authName, File dataDirectory, String defaultRealm, KDCType kdcType) {
+ this.components = components;
this.identities = identities;
this.authName = authName;
this.dataDirectory = dataDirectory;
@@ -194,11 +200,15 @@ class DeleteIdentityHandler {
commandParameters.put(KerberosServerAction.DEFAULT_REALM, defaultRealm);
commandParameters.put(KerberosServerAction.KDC_TYPE, kdcType.name());
commandParameters.put(KerberosServerAction.IDENTITY_FILTER, StageUtils.getGson().toJson(identities));
- commandParameters.put(KerberosServerAction.COMPONENT_FILTER, StageUtils.getGson().toJson(component));
+ commandParameters.put(KerberosServerAction.COMPONENT_FILTER, StageUtils.getGson().toJson(components));
commandParameters.put(KerberosServerAction.DATA_DIRECTORY, dataDirectory.getAbsolutePath());
return commandParameters;
}
+ public Set<String> getAffectedHostNames() {
+ return components.stream().map(Component::getHostName).collect(toSet());
+ }
+
public String asJson() {
return StageUtils.getGson().toJson(asMap());
}
@@ -211,22 +221,57 @@ class DeleteIdentityHandler {
processServiceComponents(
getCluster(),
kerberosDescriptor,
- Collections.singletonList(getComponentFilter()),
+ componentFilter(),
getIdentityFilter(),
dataDirectory(),
- calculateConfig(kerberosDescriptor),
- new HashMap<String, Map<String, String>>(),
+ calculateConfig(kerberosDescriptor, serviceNames()),
+ new HashMap<>(),
false,
- new HashMap<String, Set<String>>());
+ new HashMap<>());
return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", actionLog.getStdOut(), actionLog.getStdErr());
}
- protected Component getComponentFilter() {
- return StageUtils.getGson().fromJson(getCommandParameterValue(KerberosServerAction.COMPONENT_FILTER), Component.class);
+ private Set<String> serviceNames() {
+ return componentFilter().stream().map(component -> component.getServiceName()).collect(toSet());
+ }
+
+ private List<Component> componentFilter() {
+ Type jsonType = new TypeToken<List<Component>>() {}.getType();
+ return StageUtils.getGson().fromJson(getCommandParameterValue(KerberosServerAction.COMPONENT_FILTER), jsonType);
+ }
+
+ /**
+ * Cleaning identities is asynchronous, it can happen that the service and its configuration is already deleted at this point.
+ * We're extending the actual config with the properties of the latest deleted configuration of the service.
+ * The service configuration is needed because principal names may contain placeholder variables which are replaced based on the service configuration.
+ */
+ private Map<String, Map<String, String>> calculateConfig(KerberosDescriptor kerberosDescriptor, Set<String> serviceNames) throws AmbariException {
+ Map<String, Map<String, String>> actualConfig = getKerberosHelper().calculateConfigurations(getCluster(), null, kerberosDescriptor.getProperties());
+ extendWithDeletedConfigOfService(actualConfig, serviceNames);
+ return actualConfig;
+ }
+
+ private void extendWithDeletedConfigOfService(Map<String, Map<String, String>> configToBeExtended, Set<String> serviceNames) throws AmbariException {
+ Set<String> deletedConfigTypes = serviceNames.stream()
+ .flatMap(serviceName -> configTypesOfService(serviceName).stream())
+ .collect(toSet());
+ for (Config deletedConfig : getCluster().getLatestConfigsWithTypes(deletedConfigTypes)) {
+ configToBeExtended.put(deletedConfig.getType(), deletedConfig.getProperties());
+ }
}
- private Map<String, Map<String, String>> calculateConfig(KerberosDescriptor kerberosDescriptor) throws AmbariException {
- return getKerberosHelper().calculateConfigurations(getCluster(), null, kerberosDescriptor.getProperties());
+ private Set<String> configTypesOfService(String serviceName) {
+ try {
+ StackId stackId = getCluster().getCurrentStackVersion();
+ StackServiceRequest stackServiceRequest = new StackServiceRequest(stackId.getStackName(), stackId.getStackVersion(), serviceName);
+ return AmbariServer.getController().getStackServices(singleton(stackServiceRequest)).stream()
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException("Could not find stack service " + serviceName))
+ .getConfigTypes()
+ .keySet();
+ } catch (AmbariException e) {
+ throw new RuntimeException(e);
+ }
}
private String dataDirectory() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index cc0c048..3819863 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -233,7 +233,7 @@ public interface KerberosHelper {
RequestStageContainer requestStageContainer, Boolean manageIdentities)
throws AmbariException, KerberosOperationException;
- void deleteIdentity(Cluster cluster, Component component, List<String> identities) throws AmbariException, KerberosOperationException;
+ void deleteIdentities(Cluster cluster, List<Component> components, Set<String> identities) throws AmbariException, KerberosOperationException;
/**
* Updates the relevant configurations for the components specified in the service filter.
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index b30f8f6..e5b7afd 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -301,17 +301,18 @@ public class KerberosHelperImpl implements KerberosHelper {
* Deletes the kerberos identities of the given component, even if the component is already deleted.
*/
@Override
- public void deleteIdentity(Cluster cluster, Component component, List<String> identities) throws AmbariException, KerberosOperationException {
+ public void deleteIdentities(Cluster cluster, List<Component> components, Set<String> identities) throws AmbariException, KerberosOperationException {
if (identities.isEmpty()) {
return;
}
+ LOG.info("Deleting identities: ", identities);
KerberosDetails kerberosDetails = getKerberosDetails(cluster, null);
validateKDCCredentials(kerberosDetails, cluster);
File dataDirectory = createTemporaryDirectory();
RoleCommandOrder roleCommandOrder = ambariManagementController.getRoleCommandOrder(cluster);
DeleteIdentityHandler handler = new DeleteIdentityHandler(customCommandExecutionHelper, configuration.getDefaultServerTaskTimeout(), stageFactory, ambariManagementController);
DeleteIdentityHandler.CommandParams commandParameters = new DeleteIdentityHandler.CommandParams(
- component,
+ components,
identities,
ambariManagementController.getAuthName(),
dataDirectory,
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleaner.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleaner.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleaner.java
index 0a8462f..7ec4a6e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleaner.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleaner.java
@@ -17,26 +17,12 @@
*/
package org.apache.ambari.server.controller.utilities;
-import static org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptor.nullToEmpty;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.events.ServiceComponentUninstalledEvent;
+import org.apache.ambari.server.events.ServiceRemovedEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
-import org.apache.ambari.server.serveraction.kerberos.Component;
import org.apache.ambari.server.serveraction.kerberos.KerberosMissingAdminCredentialsException;
-import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.SecurityType;
-import org.apache.ambari.server.state.Service;
-import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,67 +55,29 @@ public class KerberosIdentityCleaner {
@Subscribe
public void componentRemoved(ServiceComponentUninstalledEvent event) throws KerberosMissingAdminCredentialsException {
try {
- Cluster cluster = clusters.getCluster(event.getClusterId());
- if (cluster.getSecurityType() != SecurityType.KERBEROS) {
- return;
- }
- KerberosComponentDescriptor descriptor = componentDescriptor(cluster, event.getServiceName(), event.getComponentName());
- if (descriptor == null) {
- LOG.info("No kerberos descriptor for {}", event);
- return;
- }
- List<String> identitiesToRemove = identityNames(skipSharedIdentities(descriptor.getIdentitiesSkipReferences(), cluster, event));
- LOG.info("Deleting identities {} after an event {}", identitiesToRemove, event);
- kerberosHelper.deleteIdentity(cluster, new Component(event.getHostName(), event.getServiceName(), event.getComponentName()), identitiesToRemove);
+ LOG.info("Removing identities after {}", event);
+ RemovableIdentities
+ .ofComponent(clusters.getCluster(event.getClusterId()), event, kerberosHelper)
+ .remove(kerberosHelper);
} catch (Exception e) {
LOG.error("Error while deleting kerberos identity after an event: " + event, e);
}
}
- private KerberosComponentDescriptor componentDescriptor(Cluster cluster, String serviceName, String componentName) throws AmbariException {
- KerberosServiceDescriptor serviceDescriptor = kerberosHelper.getKerberosDescriptor(cluster).getService(serviceName);
- return serviceDescriptor == null ? null : serviceDescriptor.getComponent(componentName);
- }
-
- private List<String> identityNames(List<KerberosIdentityDescriptor> identities) {
- List<String> result = new ArrayList<>();
- for (KerberosIdentityDescriptor each : identities) { result.add(each.getName()); }
- return result;
- }
-
- private List<KerberosIdentityDescriptor> skipSharedIdentities(List<KerberosIdentityDescriptor> candidates, Cluster cluster, ServiceComponentUninstalledEvent event) throws AmbariException {
- List<KerberosIdentityDescriptor> activeIdentities = activeIdentities(cluster, kerberosHelper.getKerberosDescriptor(cluster), event);
- List<KerberosIdentityDescriptor> result = new ArrayList<>();
- for (KerberosIdentityDescriptor candidate : candidates) {
- if (!candidate.isShared(activeIdentities)) {
- result.add(candidate);
- } else {
- LOG.debug("Skip removing shared identity: {}", candidate.getName());
- }
- }
- return result;
- }
-
- private List<KerberosIdentityDescriptor> activeIdentities(Cluster cluster, KerberosDescriptor root, ServiceComponentUninstalledEvent event) {
- List<KerberosIdentityDescriptor> result = new ArrayList<>();
- result.addAll(nullToEmpty(root.getIdentities()));
- for (Map.Entry<String, Service> serviceEntry : cluster.getServices().entrySet()) {
- KerberosServiceDescriptor serviceDescriptor = root.getService(serviceEntry.getKey());
- if (serviceDescriptor == null) {
- continue;
- }
- result.addAll(nullToEmpty(serviceDescriptor.getIdentities()));
- for (String componentName : serviceEntry.getValue().getServiceComponents().keySet()) {
- if (!sameComponent(event, componentName, serviceEntry.getKey())) {
- result.addAll(serviceDescriptor.getComponentIdentities(componentName));
- }
- }
+ /**
+ * Removes kerberos identities (principals and keytabs) after a service was uninstalled.
+ * Keeps the identity if either the principal or the keytab is used by an other service
+ */
+ @Subscribe
+ public void serviceRemoved(ServiceRemovedEvent event) {
+ try {
+ LOG.info("Removing identities after {}", event);
+ RemovableIdentities
+ .ofService(clusters.getCluster(event.getClusterId()), event, kerberosHelper)
+ .remove(kerberosHelper);
+ } catch (Exception e) {
+ LOG.error("Error while deleting kerberos identity after an event: " + event, e);
}
- return result;
- }
-
- private boolean sameComponent(ServiceComponentUninstalledEvent event, String componentName, String serviceName) {
- return event.getServiceName().equals(serviceName) && event.getComponentName().equals(componentName);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java
new file mode 100644
index 0000000..d4bb501
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.utilities;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.utilities.UsedIdentities.ComponentExclude;
+import org.apache.ambari.server.controller.utilities.UsedIdentities.ServiceExclude;
+import org.apache.ambari.server.events.ServiceComponentUninstalledEvent;
+import org.apache.ambari.server.events.ServiceRemovedEvent;
+import org.apache.ambari.server.serveraction.kerberos.Component;
+import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+
+/**
+ * I represent a group of kerberos identities which are to be deleted after a service or a component was removed.
+ * My instances provide methods for removing the candidates, excluding those that are still used by other components or services.
+ */
+public class RemovableIdentities {
+ private final List<KerberosIdentityDescriptor> candidateIdentities;
+ private final UsedIdentities usedIdentities;
+ private final Cluster cluster;
+ private final List<Component> components;
+
+ /**
+ * Populate the identities with the identities of the removed service and its components
+ */
+ public static RemovableIdentities ofService(Cluster cluster, ServiceRemovedEvent event, KerberosHelper kerberosHelper) throws AmbariException {
+ if (cluster.getSecurityType() != SecurityType.KERBEROS) {
+ return RemovableIdentities.none();
+ }
+ KerberosServiceDescriptor serviceDescriptor = kerberosHelper.getKerberosDescriptor(cluster).getService(event.getServiceName());
+ if (serviceDescriptor == null) {
+ return RemovableIdentities.none();
+ }
+ UsedIdentities usedIdentities = UsedIdentities.populate(cluster, excludeService(event.getServiceName()), ComponentExclude.NONE, kerberosHelper);
+ return new RemovableIdentities(
+ serviceDescriptor.getIdentitiesSkipReferences(),
+ usedIdentities,
+ cluster,
+ event.getComponents());
+ }
+
+ /**
+ * Populate the identities with the identities of the removed component
+ */
+ public static RemovableIdentities ofComponent(Cluster cluster, ServiceComponentUninstalledEvent event, KerberosHelper kerberosHelper) throws AmbariException {
+ if (cluster.getSecurityType() != SecurityType.KERBEROS) {
+ return RemovableIdentities.none();
+ }
+ KerberosServiceDescriptor serviceDescriptor = kerberosHelper.getKerberosDescriptor(cluster).getService(event.getServiceName());
+ if (serviceDescriptor == null) {
+ return RemovableIdentities.none();
+ }
+ UsedIdentities usedIdentities = UsedIdentities.populate(
+ cluster,
+ ServiceExclude.NONE,
+ excludeComponent(event.getServiceName(), event.getComponentName(), event.getHostName()),
+ kerberosHelper);
+ return new RemovableIdentities(
+ componentIdentities(singletonList(event.getComponentName()), serviceDescriptor),
+ usedIdentities,
+ cluster,
+ singletonList(event.getComponent()));
+ }
+
+ /**
+ * Populates the identities with an empty list
+ */
+ public static RemovableIdentities none() throws AmbariException {
+ return new RemovableIdentities(emptyList(), UsedIdentities.none(), null, null);
+ }
+
+ private static ServiceExclude excludeService(String excludedServiceName) {
+ return serviceName -> excludedServiceName.equals(serviceName);
+ }
+
+ private static ComponentExclude excludeComponent(String excludedServiceName, String excludedComponentName, String excludedHostName) {
+ return (serviceName, componentName, hosts) -> excludedServiceName.equals(serviceName)
+ && excludedComponentName.equals(componentName)
+ && hostNames(hosts).equals(singletonList(excludedHostName));
+ }
+
+ private static List<String> hostNames(Collection<ServiceComponentHost> hosts) {
+ return hosts.stream().map(ServiceComponentHost::getHostName).collect(toList());
+ }
+
+ private static List<KerberosIdentityDescriptor> componentIdentities(List<String> componentNames, KerberosServiceDescriptor serviceDescriptor) throws AmbariException {
+ return componentNames.stream()
+ .map(componentName -> serviceDescriptor.getComponent(componentName))
+ .filter(Objects::nonNull)
+ .flatMap(componentDescriptor -> componentDescriptor.getIdentitiesSkipReferences().stream())
+ .collect(toList());
+ }
+
+ private RemovableIdentities(List<KerberosIdentityDescriptor> candidateIdentities, UsedIdentities usedIdentities, Cluster cluster, List<Component> components) {
+ this.candidateIdentities = candidateIdentities;
+ this.usedIdentities = usedIdentities;
+ this.cluster = cluster;
+ this.components = components;
+ }
+
+ /**
+ * Remove all identities which are not used by other services or components
+ */
+ public void remove(KerberosHelper kerberosHelper) throws AmbariException, KerberosOperationException {
+ Set<String> identitiesToRemove = skipUsed().stream().map(KerberosIdentityDescriptor::getName).collect(toSet());
+ if (!identitiesToRemove.isEmpty()) {
+ kerberosHelper.deleteIdentities(cluster, components, identitiesToRemove);
+ }
+ }
+
+ private List<KerberosIdentityDescriptor> skipUsed() throws AmbariException {
+ return candidateIdentities.stream().filter(each -> !usedIdentities.contains(each)).collect(toList());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/UsedIdentities.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/UsedIdentities.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/UsedIdentities.java
new file mode 100644
index 0000000..46f5642
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/UsedIdentities.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.controller.utilities;
+
+import static java.util.Collections.emptyList;
+import static java.util.stream.Collectors.toList;
+import static org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptor.nullToEmpty;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
+import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+
+/**
+ * I represent a group of identities that are still used by any non-excluded component or service
+ */
+public class UsedIdentities {
+ private final List<KerberosIdentityDescriptor> used;
+
+ public static UsedIdentities none() throws AmbariException {
+ return new UsedIdentities(emptyList());
+ }
+
+ /**
+ * Get all identities of the installed services and components. Skip service or component that is excluded.
+ */
+ public static UsedIdentities populate(Cluster cluster, ServiceExclude serviceExclude, ComponentExclude componentExclude, KerberosHelper kerberosHelper) throws AmbariException {
+ List<KerberosIdentityDescriptor> result = new ArrayList<>();
+ KerberosDescriptor root = kerberosHelper.getKerberosDescriptor(cluster);
+ result.addAll(nullToEmpty(root.getIdentities()));
+ for (Service service : cluster.getServices().values()) {
+ if (serviceExclude.shouldExclude(service.getName())) {
+ continue;
+ }
+ KerberosServiceDescriptor serviceDescriptor = root.getService(service.getName());
+ if (serviceDescriptor != null) {
+ result.addAll(nullToEmpty(serviceDescriptor.getIdentities()));
+ result.addAll(nullToEmpty(componentIdentities(serviceDescriptor, service, componentExclude)));
+ }
+ }
+ return new UsedIdentities(result);
+ }
+
+ private static List<KerberosIdentityDescriptor> componentIdentities(KerberosServiceDescriptor serviceDescriptor, Service service, ComponentExclude componentExclude) {
+ return service.getServiceComponents().values()
+ .stream()
+ .filter(component -> !isComponentExcluded(service, componentExclude, component))
+ .flatMap(component -> serviceDescriptor.getComponentIdentities(component.getName()).stream())
+ .collect(toList());
+ }
+
+ private static boolean isComponentExcluded(Service service, ComponentExclude componentExclude, ServiceComponent component) {
+ return component.getServiceComponentHosts().isEmpty()
+ || componentExclude.shouldExclude(service.getName(), component.getName(), component.getServiceComponentHosts().values());
+ }
+
+ private UsedIdentities(List<KerberosIdentityDescriptor> used) {
+ this.used = used;
+ }
+
+ /**
+ * @return true if there is an identity in the used list with the same keytab or principal name than the given identity
+ */
+ public boolean contains(KerberosIdentityDescriptor identity) {
+ return used.stream().anyMatch(each -> identity.isShared(each));
+ }
+
+ public interface ServiceExclude {
+ boolean shouldExclude(String serviceName);
+ ServiceExclude NONE = serviceName -> false; // default implementation, exclude nothing
+ }
+
+ public interface ComponentExclude {
+ boolean shouldExclude(String serviceName, String componentName, Collection<ServiceComponentHost> hosts);
+ ComponentExclude NONE = (serviceName, componentName, hosts) -> false; // default implementation, exclude nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
index 5b55339..8acc401 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceComponentUninstalledEvent.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.events;
+import org.apache.ambari.server.serveraction.kerberos.Component;
+
/**
* The {@link ServiceComponentUninstalledEvent} class is fired when a service
* component is successfully uninstalled.
@@ -85,4 +87,8 @@ public class ServiceComponentUninstalledEvent extends ServiceEvent {
buffer.append("}");
return buffer.toString();
}
+
+ public Component getComponent() {
+ return new Component(getHostName(), getServiceName(), getComponentName());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceRemovedEvent.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceRemovedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceRemovedEvent.java
index aca00a8..de96342 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceRemovedEvent.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/events/ServiceRemovedEvent.java
@@ -17,23 +17,24 @@
*/
package org.apache.ambari.server.events;
+import static java.util.stream.Collectors.toList;
+
+import java.util.List;
+
+import org.apache.ambari.server.serveraction.kerberos.Component;
+
/**
* The {@link ServiceRemovedEvent} class is fired when a service is successfully
* removed.
*/
public class ServiceRemovedEvent extends ServiceEvent {
- /**
- * Constructor.
- *
- * @param clusterId
- * @param stackName
- * @param stackVersion
- * @param serviceName
- */
+ private final List<Component> components;
+
public ServiceRemovedEvent(long clusterId, String stackName,
- String stackVersion, String serviceName) {
+ String stackVersion, String serviceName, List<Component> components) {
super(AmbariEventType.SERVICE_REMOVED_SUCCESS, clusterId, stackName,
- stackVersion, serviceName);
+ stackVersion, serviceName);
+ this.components = components;
}
/**
@@ -49,4 +50,12 @@ public class ServiceRemovedEvent extends ServiceEvent {
buffer.append("}");
return buffer.toString();
}
+
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public List<String> getComponentNames() {
+ return components.stream().map(Component::getServiceComponentName).collect(toList());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java
index a23b914..d0f8d0b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ClusterDAO.java
@@ -217,6 +217,21 @@ public class ClusterDAO {
}
/**
+ * Gets the latest configurations for a given stack with any of the given config types.
+ * This method does not take into account the configuration being enabled.
+ */
+ @RequiresSession
+ public List<ClusterConfigEntity> getLatestConfigurationsWithTypes(long clusterId, StackId stackId, Collection<String> configTypes) {
+ StackEntity stackEntity = stackDAO.find(stackId.getStackName(), stackId.getStackVersion());
+ return daoUtils.selectList(
+ entityManagerProvider.get()
+ .createNamedQuery("ClusterConfigEntity.findLatestConfigsByStackWithTypes", ClusterConfigEntity.class)
+ .setParameter("clusterId", clusterId)
+ .setParameter("stack", stackEntity)
+ .setParameter("types", configTypes));
+ }
+
+ /**
* Gets the latest configurations for a given stack for all of the
* configurations of the specified cluster.
*
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
index 34f3034..3a74367 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
@@ -62,6 +62,9 @@ import org.apache.commons.lang.builder.EqualsBuilder;
name = "ClusterConfigEntity.findLatestConfigsByStack",
query = "SELECT clusterConfig FROM ClusterConfigEntity clusterConfig WHERE clusterConfig.clusterId = :clusterId AND clusterConfig.stack = :stack AND clusterConfig.selectedTimestamp = (SELECT MAX(clusterConfig2.selectedTimestamp) FROM ClusterConfigEntity clusterConfig2 WHERE clusterConfig2.clusterId=:clusterId AND clusterConfig2.stack=:stack AND clusterConfig2.type = clusterConfig.type)"),
@NamedQuery(
+ name = "ClusterConfigEntity.findLatestConfigsByStackWithTypes",
+ query = "SELECT clusterConfig FROM ClusterConfigEntity clusterConfig WHERE clusterConfig.type IN :types AND clusterConfig.clusterId = :clusterId AND clusterConfig.stack = :stack AND clusterConfig.selectedTimestamp = (SELECT MAX(clusterConfig2.selectedTimestamp) FROM ClusterConfigEntity clusterConfig2 WHERE clusterConfig2.clusterId=:clusterId AND clusterConfig2.stack=:stack AND clusterConfig2.type = clusterConfig.type)"),
+ @NamedQuery(
name = "ClusterConfigEntity.findNotMappedClusterConfigsToService",
query = "SELECT clusterConfig FROM ClusterConfigEntity clusterConfig WHERE clusterConfig.serviceConfigEntities IS EMPTY AND clusterConfig.type != 'cluster-env'"),
@NamedQuery(
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
index b4f7120..9597ba1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java
@@ -267,6 +267,13 @@ public interface Cluster {
Config getConfig(String configType, String versionTag);
/**
+ * Get latest (including inactive ones) configurations with any of the given types.
+ * This method does not take into account the configuration being enabled.
+ * @return the list of configurations with the given types
+ */
+ List<Config> getLatestConfigsWithTypes(Collection<String> types);
+
+ /**
* Gets the specific config that matches the specified type and version. This not
* necessarily a DESIRED configuration that applies to a cluster.
* @param configType the config type to find
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
index 5084703..74d79c8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
@@ -51,6 +51,7 @@ import org.apache.ambari.server.orm.entities.ServiceConfigEntity;
import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntityPK;
import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.serveraction.kerberos.Component;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -588,6 +589,7 @@ public class ServiceImpl implements Service {
@Override
@Transactional
public void delete() throws AmbariException {
+ List<Component> components = getComponents(); // XXX temporal coupling, need to call this BEFORE deletingAllComponents
deleteAllComponents();
deleteAllServiceConfigs();
@@ -601,11 +603,21 @@ public class ServiceImpl implements Service {
}
ServiceRemovedEvent event = new ServiceRemovedEvent(getClusterId(), stackId.getStackName(),
- stackId.getStackVersion(), getName());
+ stackId.getStackVersion(), getName(), components);
eventPublisher.publish(event);
}
+ private List<Component> getComponents() {
+ List<Component> result = new ArrayList<>();
+ for (ServiceComponent component : getServiceComponents().values()) {
+ for (ServiceComponentHost host : component.getServiceComponentHosts().values()) {
+ result.add(new Component(host.getHostName(), getName(), component.getName()));
+ }
+ }
+ return result;
+ }
+
@Transactional
protected void removeEntities() throws AmbariException {
serviceDesiredStateDAO.removeByPK(serviceDesiredStateEntityPK);
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 06b6217..c950d67 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReadWriteLock;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.persistence.EntityManager;
@@ -1125,6 +1126,14 @@ public class ClusterImpl implements Cluster {
}
@Override
+ public List<Config> getLatestConfigsWithTypes(Collection<String> types) {
+ return clusterDAO.getLatestConfigurationsWithTypes(clusterId, getDesiredStackVersion(), types)
+ .stream()
+ .map(clusterConfigEntity -> configFactory.createExisting(this, clusterConfigEntity))
+ .collect(Collectors.toList());
+ }
+
+ @Override
public Config getConfigByVersion(String configType, Long configVersion) {
clusterGlobalLock.readLock().lock();
try {
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
index 0a89c1d..5658133 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
@@ -18,6 +18,8 @@
package org.apache.ambari.server.state.kerberos;
+import static java.util.stream.Collectors.toList;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -777,6 +779,16 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
return map;
}
+ /**
+ * @return identities which are not references to other identities
+ */
+ public List<KerberosIdentityDescriptor> getIdentitiesSkipReferences() {
+ return nullToEmpty(getIdentities())
+ .stream()
+ .filter(identity -> !identity.getReferencedServiceName().isPresent() && identity.getName() != null && !identity.getName().startsWith("/"))
+ .collect(toList());
+ }
+
@Override
public int hashCode() {
return super.hashCode() +
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
index 41d1f65..768a17e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
@@ -17,9 +17,7 @@
*/
package org.apache.ambari.server.state.kerberos;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.List;
import java.util.Map;
/**
@@ -113,19 +111,6 @@ public class KerberosComponentDescriptor extends AbstractKerberosDescriptorConta
return null;
}
- /**
- * @return identities which are not references to other identities
- */
- public List<KerberosIdentityDescriptor> getIdentitiesSkipReferences() {
- List<KerberosIdentityDescriptor> result = new ArrayList<>();
- for (KerberosIdentityDescriptor each : nullToEmpty(getIdentities())) {
- if (!each.getReferencedServiceName().isPresent() && each.getName() != null && !each.getName().startsWith("/")) {
- result.add(each);
- }
- }
- return result;
- }
-
@Override
public int hashCode() {
return 35 * super.hashCode();
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
index 2023793..911723b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
@@ -17,10 +17,8 @@
*/
package org.apache.ambari.server.state.kerberos;
-import java.util.List;
import java.util.Map;
-import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.collections.Predicate;
import org.apache.ambari.server.collections.PredicateUtils;
@@ -371,16 +369,12 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
}
}
+
/**
- * @return true if this identity either has the same principal or keytab as any of the given identities.
+ * @return true if the given identity has the same principal or keytab as me
*/
- public boolean isShared(List<KerberosIdentityDescriptor> identities) throws AmbariException {
- for (KerberosIdentityDescriptor each : identities) {
- if (hasSamePrincipal(each) || hasSameKeytab(each)) {
- return true;
- }
- }
- return false;
+ public boolean isShared(KerberosIdentityDescriptor that) {
+ return hasSamePrincipal(that) || hasSameKeytab(that);
}
private boolean hasSameKeytab(KerberosIdentityDescriptor that) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java
index d22c92e..027f339 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java
@@ -18,15 +18,20 @@
package org.apache.ambari.server.controller.utilities;
import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.newHashSet;
+import static java.util.Collections.singletonList;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.reset;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.ambari.server.controller.KerberosHelper;
import org.apache.ambari.server.events.ServiceComponentUninstalledEvent;
+import org.apache.ambari.server.events.ServiceRemovedEvent;
import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.serveraction.kerberos.Component;
import org.apache.ambari.server.serveraction.kerberos.KerberosMissingAdminCredentialsException;
@@ -35,6 +40,7 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.SecurityType;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
+import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
import org.easymock.EasyMockRule;
@@ -47,6 +53,7 @@ import org.junit.Test;
public class KerberosIdentityCleanerTest extends EasyMockSupport {
@Rule public EasyMockRule mocks = new EasyMockRule(this);
private static final String HOST = "c6401";
+ private static final String HOST2 = "c6402";
private static final String OOZIE = "OOZIE";
private static final String OOZIE_SERVER = "OOZIE_SERVER";
private static final String OOZIE_2 = "OOZIE2";
@@ -55,6 +62,9 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
private static final String RESOURCE_MANAGER_2 = "RESOURCE_MANAGER2";
private static final String YARN = "YARN";
private static final String RESOURCE_MANAGER = "RESOURCE_MANAGER";
+ private static final String HDFS = "HDFS";
+ private static final String NAMENODE = "NAMENODE";
+ private static final String DATANODE = "DATANODE";
private static final long CLUSTER_ID = 1;
@Mock private KerberosHelper kerberosHelper;
@Mock private Clusters clusters;
@@ -66,8 +76,8 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
@Test
public void removesAllKerberosIdentitesOfComponentAfterComponentWasUninstalled() throws Exception {
- installComponent(OOZIE, OOZIE_SERVER);
- kerberosHelper.deleteIdentity(cluster, new Component(HOST, OOZIE, OOZIE_SERVER), newArrayList("oozie_server1", "oozie_server2"));
+ installComponent(OOZIE, OOZIE_SERVER, HOST);
+ kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("oozie_server1", "oozie_server2"));
expectLastCall().once();
replayAll();
uninstallComponent(OOZIE, OOZIE_SERVER, HOST);
@@ -83,9 +93,9 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
@Test
public void skipsRemovingIdentityThatIsSharedByPrincipalName() throws Exception {
- installComponent(OOZIE, OOZIE_SERVER);
- installComponent(OOZIE_2, OOZIE_SERVER_2);
- kerberosHelper.deleteIdentity(cluster, new Component(HOST, OOZIE, OOZIE_SERVER), newArrayList("oozie_server1"));
+ installComponent(OOZIE, OOZIE_SERVER, HOST);
+ installComponent(OOZIE_2, OOZIE_SERVER_2, HOST);
+ kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("oozie_server1"));
expectLastCall().once();
replayAll();
uninstallComponent(OOZIE, OOZIE_SERVER, HOST);
@@ -94,9 +104,9 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
@Test
public void skipsRemovingIdentityThatIsSharedByKeyTabFilePath() throws Exception {
- installComponent(YARN, RESOURCE_MANAGER);
- installComponent(YARN_2, RESOURCE_MANAGER_2);
- kerberosHelper.deleteIdentity(cluster, new Component(HOST, YARN, RESOURCE_MANAGER), newArrayList("rm_unique"));
+ installComponent(YARN, RESOURCE_MANAGER, HOST);
+ installComponent(YARN_2, RESOURCE_MANAGER_2, HOST);
+ kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, YARN, RESOURCE_MANAGER)), newHashSet("rm_unique"));
expectLastCall().once();
replayAll();
uninstallComponent(YARN, RESOURCE_MANAGER, HOST);
@@ -112,11 +122,43 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
verifyAll();
}
- private void installComponent(String serviceName, final String componentName) {
+ @Test
+ public void skipsRemovingIdentityIfComponentIsStillInstalledOnADifferentHost() throws Exception {
+ installComponent(OOZIE, OOZIE_SERVER, HOST, HOST2);
+ replayAll();
+ uninstallComponent(OOZIE, OOZIE_SERVER, HOST);
+ verifyAll();
+ }
+
+ @Test
+ public void removesServiceIdentitiesSkipComponentIdentitiesAfterServiceWasUninstalled() throws Exception {
+ installComponent(OOZIE, OOZIE_SERVER, HOST);
+ kerberosHelper.deleteIdentities(cluster, hdfsComponents(), newHashSet("hdfs-service"));
+ expectLastCall().once();
+ replayAll();
+ uninstallService(HDFS, hdfsComponents());
+ verifyAll();
+ }
+
+ private ArrayList<Component> hdfsComponents() {
+ return newArrayList(new Component(HOST, HDFS, NAMENODE), new Component(HOST, HDFS, DATANODE));
+ }
+
+ private void installComponent(String serviceName, String componentName, String... hostNames) {
Service service = createMock(serviceName + "_" + componentName, Service.class);
+ ServiceComponent component = createMock(componentName, ServiceComponent.class);
+ expect(component.getName()).andReturn(componentName).anyTimes();
+ Map<String, ServiceComponentHost> hosts = new HashMap<>();
+ expect(component.getServiceComponentHosts()).andReturn(hosts).anyTimes();
+ for (String hostName : hostNames) {
+ ServiceComponentHost host = createMock(hostName, ServiceComponentHost.class);
+ expect(host.getHostName()).andReturn(hostName).anyTimes();
+ hosts.put(hostName, host);
+ }
installedServices.put(serviceName, service);
+ expect(service.getName()).andReturn(serviceName).anyTimes();
expect(service.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>() {{
- put(componentName, null);
+ put(componentName, component);
}}).anyTimes();
}
@@ -124,6 +166,10 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
kerberosIdentityCleaner.componentRemoved(new ServiceComponentUninstalledEvent(CLUSTER_ID, "any", "any", service, component, host, false));
}
+ private void uninstallService(String service, List<Component> components) throws KerberosMissingAdminCredentialsException {
+ kerberosIdentityCleaner.serviceRemoved(new ServiceRemovedEvent(CLUSTER_ID, "any", "any", service, components));
+ }
+
@Before
public void setUp() throws Exception {
kerberosIdentityCleaner = new KerberosIdentityCleaner(new AmbariEventPublisher(), kerberosHelper, clusters);
@@ -139,7 +185,8 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
" 'name': '/HDFS/NAMENODE/hdfs'" +
" }," +
" {" +
- " 'name': 'oozie_server1'" +
+ " 'name': 'oozie_server1'," +
+ " 'principal': { 'value': 'oozie1/_HOST@EXAMPLE.COM' }" +
" }," +"" +
" {" +
" 'name': 'oozie_server2'," +
@@ -193,6 +240,39 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport {
" ]" +
" }" +
" ]" +
+ " }," +
+ " {" +
+ " 'name': 'HDFS'," +
+ " 'identities': [" +
+ " {" +
+ " 'name': 'hdfs-service'" +
+ " }," +
+ " {" +
+ " 'name': 'shared'," +
+ " 'principal': { 'value': 'oozie/_HOST@EXAMPLE.COM' }" +
+ " }," +
+ " {" +
+ " 'name': '/YARN/RESOURCE_MANAGER/rm'" +
+ " }," +
+ " ]," +
+ " 'components': [" +
+ " {" +
+ " 'name': 'NAMENODE'," +
+ " 'identities': [" +
+ " {" +
+ " 'name': 'namenode'" +
+ " }" +
+ " ]" +
+ " }," +
+ " {" +
+ " 'name': 'DATANODE'," +
+ " 'identities': [" +
+ " {" +
+ " 'name': 'datanode'" +
+ " }" +
+ " ]" +
+ " }" +
+ " ]" +
" }" +
" ]" +
"}");
http://git-wip-us.apache.org/repos/asf/ambari/blob/e767aa44/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/ServiceConfigDAOTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/ServiceConfigDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/ServiceConfigDAOTest.java
index 406349a..80cb4dc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/ServiceConfigDAOTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/ServiceConfigDAOTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.ambari.server.orm.dao;
+import static java.util.Arrays.asList;
+
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
@@ -482,6 +484,16 @@ public class ServiceConfigDAOTest {
Assert.assertTrue(entity.isSelected());
}
+ @Test
+ public void testGetLatestClusterConfigsWithTypes() throws Exception {
+ initClusterEntities();
+ ClusterEntity clusterEntity = clusterDAO.findByName("c1");
+ List<ClusterConfigEntity> entities = clusterDAO.getLatestConfigurationsWithTypes(clusterEntity.getClusterId(), HDP_01, asList("oozie-site"));
+ Assert.assertEquals(1, entities.size());
+ entities = clusterDAO.getLatestConfigurationsWithTypes(clusterEntity.getClusterId(), HDP_01, asList("no-such-type"));
+ Assert.assertTrue(entities.isEmpty());
+ }
+
/**
* Tests getting latest and enabled configurations when there is a
* configuration group. Configurations for configuration groups are not
[2/6] ambari git commit: AMBARI-21435 Add python functions to get the
Ambari version of the agent (mgergely)
Posted by rl...@apache.org.
AMBARI-21435 Add python functions to get the Ambari version of the agent (mgergely)
Change-Id: I9481b32babac92ad5d7496fe4abb208eefaac922
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/383b8c7d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/383b8c7d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/383b8c7d
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 383b8c7d83545bc4cb21058794e91b5a4aece425
Parents: e767aa4
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Jul 12 13:22:33 2017 +0200
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Wed Jul 12 13:22:33 2017 +0200
----------------------------------------------------------------------
.../python/ambari_commons/get_ambari_version.py | 44 ++++++++++++++++++++
1 file changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/383b8c7d/ambari-common/src/main/python/ambari_commons/get_ambari_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/get_ambari_version.py b/ambari-common/src/main/python/ambari_commons/get_ambari_version.py
new file mode 100644
index 0000000..c8c5336
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_commons/get_ambari_version.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import ConfigParser
+from resource_management.core.logger import Logger
+
+"""
+returns the ambari version on an agent host
+"""
+def get_ambari_version_agent():
+ ambari_version = None
+ AMBARI_AGENT_CONF = '/etc/ambari-agent/conf/ambari-agent.ini'
+ if os.path.exists(AMBARI_AGENT_CONF):
+ try:
+ ambari_agent_config = ConfigParser.RawConfigParser()
+ ambari_agent_config.read(AMBARI_AGENT_CONF)
+ data_dir = ambari_agent_config.get('agent', 'prefix')
+ ver_file = os.path.join(data_dir, 'version')
+ with open(ver_file, "r") as f:
+ ambari_version = f.read().strip()
+ except Exception, e:
+ Logger.info('Unable to determine ambari version from the agent version file.')
+ Logger.debug('Exception: %s' % str(e))
+ pass
+ pass
+ return ambari_version
[5/6] ambari git commit: AMBARI-21445. Fixes the following bugs :
(1). Make Hive Kerberos keytab files group non-readable (2). HiveServer2
Authentication via LDAP to work correctly (3). Remove leading while spaces
for the hive-env and hive-interactive-en
Posted by rl...@apache.org.
AMBARI-21445. Fixes the following bugs : (1). Make Hive Kerberos keytab files group non-readable (2). HiveServer2 Authentication via LDAP to work correctly (3). Remove leading while spaces for the hive-env and hive-interactive-env template.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/eb3d3ea6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/eb3d3ea6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/eb3d3ea6
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: eb3d3ea6e5eb9464a135f851658d4aa5b3988efa
Parents: 9f788c3
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Tue Jul 11 15:37:08 2017 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Wed Jul 12 11:55:44 2017 -0700
----------------------------------------------------------------------
.../0.12.0.2.0/package/scripts/params_linux.py | 4 +
.../0.12.0.2.0/package/scripts/service_check.py | 3 +-
.../services/HIVE/configuration/hive-env.xml | 78 +++++-----
.../HIVE/configuration/hive-interactive-env.xml | 62 ++++----
.../stacks/HDP/2.6/services/HIVE/kerberos.json | 151 +++++++++++++++++++
.../stacks/HDP/2.6/services/YARN/kerberos.json | 2 +-
6 files changed, 228 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 21b3d8b..9939536 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -849,3 +849,7 @@ ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-prope
if security_enabled:
hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
+
+# For ldap - hive_check
+hive_ldap_user= config['configurations']['hive-env'].get('alert_ldap_username','')
+hive_ldap_passwd=config['configurations']['hive-env'].get('alert_ldap_password','')
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
index d144c34..271fff9 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
@@ -123,7 +123,8 @@ class HiveServiceCheckDefault(HiveServiceCheck):
params.hive_server_principal, kinit_cmd, params.smokeuser,
transport_mode=params.hive_transport_mode, http_endpoint=params.hive_http_endpoint,
ssl=params.hive_ssl, ssl_keystore=ssl_keystore,
- ssl_password=ssl_password)
+ ssl_password=ssl_password, ldap_username=params.hive_ldap_user,
+ ldap_password=params.hive_ldap_passwd)
Logger.info("Successfully connected to {0} on port {1}".format(address, server_port))
workable_server_available = True
except:
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml
index a6cf1bc..929c10d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-env.xml
@@ -60,56 +60,56 @@
<display-name>hive-env template</display-name>
<description>This is the jinja template for hive-env.sh file</description>
<value>
- export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB
- if [ "$SERVICE" = "cli" ]; then
- if [ -z "$DEBUG" ]; then
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit"
- else
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
- fi
- fi
+export HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB
+if [ "$SERVICE" = "cli" ]; then
+ if [ -z "$DEBUG" ]; then
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit"
+ else
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
+ fi
+fi
- # The heap size of the jvm stared by hive shell script can be controlled via:
+# The heap size of the jvm stared by hive shell script can be controlled via:
- if [ "$SERVICE" = "metastore" ]; then
- export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
- else
- export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
- fi
+if [ "$SERVICE" = "metastore" ]; then
+ export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
+else
+ export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client
+fi
- export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
- export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}"
- # Larger heap size may be required when running queries over large number of files or partitions.
- # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
- # appropriate for hive server (hwi etc).
+# Larger heap size may be required when running queries over large number of files or partitions.
+# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
+# appropriate for hive server (hwi etc).
- # Set HADOOP_HOME to point to a specific hadoop install directory
- HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+# Set HADOOP_HOME to point to a specific hadoop install directory
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
- export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}}
+export HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}}
- # Hive Configuration Directory can be controlled by:
- export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}}
+# Hive Configuration Directory can be controlled by:
+export HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}}
- # Folder containing extra libraries required for hive compilation/execution can be controlled by:
- if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
- if [ -f "${HIVE_AUX_JARS_PATH}" ]; then
- export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
- elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then
- export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar
- fi
- elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then
- export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar
- fi
+# Folder containing extra libraries required for hive compilation/execution can be controlled by:
+if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
+ if [ -f "${HIVE_AUX_JARS_PATH}" ]; then
+ export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
+ elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then
+ export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar
+ fi
+elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then
+ export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar
+fi
- export METASTORE_PORT={{hive_metastore_port}}
+export METASTORE_PORT={{hive_metastore_port}}
- {% if sqla_db_used or lib_dir_available %}
- export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}"
- export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
- {% endif %}
+{% if sqla_db_used or lib_dir_available %}
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}"
+export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
+{% endif %}
</value>
<value-attributes>
<type>content</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
index ada4859..86720f4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-env.xml
@@ -100,47 +100,47 @@
<display-name>hive-interactive-env template</display-name>
<description>This is the jinja template for hive-env.sh file</description>
<value>
- if [ "$SERVICE" = "cli" ]; then
- if [ -z "$DEBUG" ]; then
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit"
- else
- export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
- fi
- fi
+if [ "$SERVICE" = "cli" ]; then
+ if [ -z "$DEBUG" ]; then
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit"
+ else
+ export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit"
+ fi
+fi
- # The heap size of the jvm stared by hive shell script can be controlled via:
+# The heap size of the jvm stared by hive shell script can be controlled via:
- if [ "$SERVICE" = "metastore" ]; then
- export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
- else
- export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client
- fi
+if [ "$SERVICE" = "metastore" ]; then
+ export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore
+else
+ export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client
+fi
- export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
- export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS{{heap_dump_opts}}"
- # Larger heap size may be required when running queries over large number of files or partitions.
- # By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
- # appropriate for hive server (hwi etc).
+# Larger heap size may be required when running queries over large number of files or partitions.
+# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be
+# appropriate for hive server (hwi etc).
- # Set HADOOP_HOME to point to a specific hadoop install directory
- HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+# Set HADOOP_HOME to point to a specific hadoop install directory
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
- # Hive Configuration Directory can be controlled by:
- export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}}
+# Hive Configuration Directory can be controlled by:
+export HIVE_CONF_DIR={{hive_server_interactive_conf_dir}}
- # Add additional hcatalog jars
- if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
- export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
- else
- export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar
- fi
+# Add additional hcatalog jars
+if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then
+ export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}
+else
+ export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar
+fi
- export METASTORE_PORT={{hive_metastore_port}}
+export METASTORE_PORT={{hive_metastore_port}}
- # Spark assembly contains a conflicting copy of HiveConf from hive-1.2
- export HIVE_SKIP_SPARK_ASSEMBLY=true
+# Spark assembly contains a conflicting copy of HiveConf from hive-1.2
+export HIVE_SKIP_SPARK_ASSEMBLY=true
</value>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json
new file mode 100644
index 0000000..b6e57e1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/kerberos.json
@@ -0,0 +1,151 @@
+{
+ "services": [
+ {
+ "name": "HIVE",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "configurations": [
+ {
+ "hive-site": {
+ "hive.metastore.sasl.enabled": "true",
+ "hive.server2.authentication": "KERBEROS"
+ }
+ },
+ {
+ "ranger-hive-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "HIVE_METASTORE",
+ "identities": [
+ {
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-site/hive.metastore.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "hive-site/hive.metastore.kerberos.keytab.file"
+ }
+ }
+ ]
+ },
+ {
+ "name": "HIVE_SERVER",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "hive_server_hive",
+ "principal": {
+ "value": "hive/_HOST@${realm}",
+ "type": "service",
+ "configuration": "hive-site/hive.server2.authentication.kerberos.principal",
+ "local_username": "${hive-env/hive_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.service.keytab",
+ "owner": {
+ "name": "${hive-env/hive_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "hive-site/hive.server2.authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "atlas_kafka",
+ "reference": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.principal"
+ },
+ "keytab": {
+ "configuration": "hive-atlas-application.properties/atlas.jaas.KafkaClient.option.keyTab"
+ }
+ },
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "hive-site/hive.server2.authentication.spnego.principal"
+ },
+ "keytab": {
+ "configuration": "hive-site/hive.server2.authentication.spnego.keytab"
+ }
+ },
+ {
+ "name": "ranger_audit",
+ "reference": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "ranger-hive-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-hive-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
+ }
+ ]
+ },
+ {
+ "name": "HIVE_SERVER_INTERACTIVE",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/hive_server_hive"
+ },
+ {
+ "name": "/HIVE/HIVE_SERVER/spnego"
+ },
+ {
+ "name": "/YARN/NODEMANAGER/llap_zk_hive"
+ }
+ ]
+ },
+ {
+ "name": "WEBHCAT_SERVER",
+ "identities": [
+ {
+ "name": "/spnego",
+ "principal": {
+ "configuration": "webhcat-site/templeton.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "webhcat-site/templeton.kerberos.keytab"
+ }
+ }
+ ],
+ "configurations": [
+ {
+ "core-site": {
+ "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host|append(core-site/hadoop.proxyuser.HTTP.hosts, \\\\,, true)}"
+ }
+ },
+ {
+ "webhcat-site": {
+ "templeton.kerberos.secret": "secret",
+ "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/eb3d3ea6/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json
index b1501b8..60d50eb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/kerberos.json
@@ -117,7 +117,7 @@
},
"group": {
"name": "${cluster-env/user_group}",
- "access": "r"
+ "access": ""
},
"configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file"
},
[6/6] ambari git commit: Merge branch 'trunk' into
branch-feature-AMBARI-20859
Posted by rl...@apache.org.
Merge branch 'trunk' into branch-feature-AMBARI-20859
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a0850b4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a0850b4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a0850b4
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 7a0850b49577dea155ab76b4e7a13b3bb2d5291c
Parents: 7097109 eb3d3ea
Author: Robert Levas <rl...@hortonworks.com>
Authored: Wed Jul 12 17:00:45 2017 -0400
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Wed Jul 12 17:00:45 2017 -0400
----------------------------------------------------------------------
.../python/ambari_commons/get_ambari_version.py | 44 ++++++
.../ambari/logfeeder/mapper/MapperDate.java | 42 ++++--
.../controller/DeleteIdentityHandler.java | 77 ++++++++--
.../server/controller/KerberosHelper.java | 2 +-
.../server/controller/KerberosHelperImpl.java | 5 +-
.../utilities/KerberosIdentityCleaner.java | 88 +++--------
.../utilities/RemovableIdentities.java | 145 ++++++++++++++++++
.../controller/utilities/UsedIdentities.java | 101 +++++++++++++
.../ServiceComponentUninstalledEvent.java | 6 +
.../server/events/ServiceRemovedEvent.java | 29 ++--
.../ambari/server/orm/dao/ClusterDAO.java | 15 ++
.../orm/entities/ClusterConfigEntity.java | 3 +
.../org/apache/ambari/server/state/Cluster.java | 7 +
.../apache/ambari/server/state/ServiceImpl.java | 14 +-
.../server/state/cluster/ClusterImpl.java | 9 ++
.../AbstractKerberosDescriptorContainer.java | 12 ++
.../kerberos/KerberosComponentDescriptor.java | 15 --
.../kerberos/KerberosIdentityDescriptor.java | 14 +-
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 6 +
.../0.12.0.2.0/package/scripts/service_check.py | 3 +-
.../services/HIVE/configuration/hive-env.xml | 78 +++++-----
.../HIVE/configuration/hive-interactive-env.xml | 62 ++++----
.../services/HIVE/configuration/hive-site.xml | 35 +++++
.../stacks/HDP/2.6/services/HIVE/kerberos.json | 151 +++++++++++++++++++
.../stacks/HDP/2.6/services/YARN/kerberos.json | 2 +-
.../utilities/KerberosIdentityCleanerTest.java | 102 +++++++++++--
.../server/orm/dao/ServiceConfigDAOTest.java | 12 ++
.../stacks/2.0.6/HIVE/test_hive_server.py | 2 +
29 files changed, 861 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
[4/6] ambari git commit: AMBARI-21447 Log Feeder should support logs
without date (time only) (mgergely)
Posted by rl...@apache.org.
AMBARI-21447 Log Feeder should support logs without date (time only) (mgergely)
Change-Id: I853447134873b10fdd3fd604fd84630a9caf9d03
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f788c38
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f788c38
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f788c38
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 9f788c386667bfeb82fff7c35287a5fdb175c349
Parents: 31b9d77
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Jul 12 16:55:48 2017 +0200
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Wed Jul 12 16:55:48 2017 +0200
----------------------------------------------------------------------
.../ambari/logfeeder/mapper/MapperDate.java | 42 +++++++++++++-------
1 file changed, 28 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f788c38/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 305688b..e099161 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.mapper;
+import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
@@ -79,20 +80,7 @@ public class MapperDate extends Mapper {
jsonObj.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, ((Date) value).getTime());
} else if (targetDateFormatter != null) {
if (srcDateFormatter != null) {
- Date srcDate = srcDateFormatter.parse(value.toString());
- //set year in src_date when src_date does not have year component
- if (!srcDateFormatter.toPattern().contains("yy")) {
- Calendar currentCalendar = Calendar.getInstance();
- Calendar logDateCalendar = Calendar.getInstance();
- logDateCalendar.setTimeInMillis(srcDate.getTime());
- if (logDateCalendar.get(Calendar.MONTH) > currentCalendar.get(Calendar.MONTH)) {
- // set previous year as a log year when log month is grater than current month
- srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR) - 1);
- } else {
- // set current year as a log year
- srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR));
- }
- }
+ Date srcDate = getSourceDate(value);
value = targetDateFormatter.format(srcDate);
jsonObj.put(LogFeederConstants.IN_MEMORY_TIMESTAMP, srcDate.getTime());
} else {
@@ -111,4 +99,30 @@ public class MapperDate extends Mapper {
}
return value;
}
+
+ private Date getSourceDate(Object value) throws ParseException {
+ Date srcDate = srcDateFormatter.parse(value.toString());
+
+ Calendar currentCalendar = Calendar.getInstance();
+
+ if (!srcDateFormatter.toPattern().contains("dd")) {
+ //set year/month/date in src_date when src_date does not have date component
+ srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR));
+ srcDate = DateUtils.setMonths(srcDate, currentCalendar.get(Calendar.MONTH));
+ srcDate = DateUtils.setDays(srcDate, currentCalendar.get(Calendar.DAY_OF_MONTH));
+ // if with the current date the time stamp is after the current one, it must be previous day
+ if (srcDate.getTime() > currentCalendar.getTimeInMillis()) {
+ srcDate = DateUtils.addDays(srcDate, -1);
+ }
+ } else if (!srcDateFormatter.toPattern().contains("yy")) {
+ //set year in src_date when src_date does not have year component
+ srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR));
+ // if with the current year the time stamp is after the current one, it must be previous year
+ if (srcDate.getTime() > currentCalendar.getTimeInMillis()) {
+ srcDate = DateUtils.addYears(srcDate, -1);
+ }
+ }
+
+ return srcDate;
+ }
}
[3/6] ambari git commit: AMBARI-21444. Hive warehouse
fixes.(vbrodetskyi)
Posted by rl...@apache.org.
AMBARI-21444. Hive warehouse fixes.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31b9d777
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31b9d777
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31b9d777
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 31b9d7774b22f59a4d7120c9836c73a5216fd529
Parents: 383b8c7
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Wed Jul 12 15:35:53 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Wed Jul 12 15:35:53 2017 +0300
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 2 ++
.../services/HIVE/configuration/hive-site.xml | 35 ++++++++++++++++++++
.../stacks/2.0.6/HIVE/test_hive_server.py | 2 ++
4 files changed, 41 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 36725c3..8e176b6 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -224,7 +224,8 @@ def setup_hiveserver2():
type="directory",
action="create_on_execute",
owner=params.hive_user,
- mode=0777
+ group=params.user_group,
+ mode=params.hive_apps_whs_mode
)
else:
Logger.info(format("Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."))
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 078076a..21b3d8b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -505,6 +505,8 @@ hive_env_sh_template = config['configurations']['hive-env']['content']
hive_hdfs_user_dir = format("/user/{hive_user}")
hive_hdfs_user_mode = 0755
+#Parameter for custom warehouse directory permissions. Permissions are in octal format and need to be converted to decimal
+hive_apps_whs_mode = int(default('/configurations/hive-site/custom.hive.warehouse.mode', '0777'), 8)
hive_apps_whs_dir = config['configurations']['hive-site']["hive.metastore.warehouse.dir"]
whs_dir_protocol = urlparse(hive_apps_whs_dir).scheme
hive_exec_scratchdir = config['configurations']['hive-site']["hive.exec.scratchdir"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
new file mode 100644
index 0000000..a07c16f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration supports_final="true">
+ <property>
+ <name>hive.warehouse.subdir.inherit.perms</name>
+ <value>true</value>
+ <description>Set this to true if table directories should inherit the permissions of the warehouse or database directory instead of being created with permissions derived from dfs umask
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.start.cleanup.scratchdir</name>
+ <value>false</value>
+ <description>To cleanup the hive scratchdir while starting the hive server.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index ae2ec86..fc6d14e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -491,6 +491,7 @@ class TestHiveServer(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='missing_principal', default_fs=default_fs_default,
+ group = 'hadoop',
mode = 0777,
)
self.assertResourceCalled('HdfsResource', '/user/hive',
@@ -703,6 +704,7 @@ class TestHiveServer(RMFTestCase):
hadoop_conf_dir = '/etc/hadoop/conf',
type = 'directory',
action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020',
+ group = 'hadoop',
mode = 0777,
)
self.assertResourceCalled('HdfsResource', '/user/hive',