You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2017/10/11 08:38:43 UTC
[5/7] ambari git commit: AMBARI-22190. After merging trunk to
branch-3.0-perf some parts of code are missing. (mpapirkovskyy)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 3688727..6dc2b93 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -19,6 +19,8 @@
package org.apache.ambari.server.stack;
import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -1143,10 +1145,52 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
stackInfo.getRepositories().addAll(serviceRepos);
if (null != rxml && null != rxml.getLatestURI() && stackRepos.size() > 0) {
- stackContext.registerRepoUpdateTask(rxml.getLatestURI(), this);
+ registerRepoUpdateTask(rxml);
}
}
+ private void registerRepoUpdateTask(RepositoryXml rxml) {
+ String latest = rxml.getLatestURI();
+ if (StringUtils.isBlank(latest)) {
+ return;
+ }
+
+ URI uri = getURI(this, latest);
+
+ if (null == uri) {
+ LOG.warn("Could not determine how to load stack {}-{} latest definition for {}",
+ stackInfo.getName(), stackInfo.getVersion(), latest);
+ return;
+ }
+
+ stackContext.registerRepoUpdateTask(uri, this);
+ }
+
+ /**
+ * @param module
+ * the stack module
+ * @param uriString
+ * the uri string
+ * @return a repo URI, even if it is relative-file based
+ */
+ public static URI getURI(StackModule module, String uriString) {
+
+ URI uri = null;
+ if (uriString.startsWith("http")) {
+ try {
+ uri = new URI(uriString);
+ } catch (URISyntaxException e) {
+ // should be logged later
+ }
+ } else if ('.' == uriString.charAt(0)) {
+ uri = new File(module.getStackDirectory().getRepoDir(), uriString).toURI();
+ } else {
+ uri = new File(uriString).toURI();
+ }
+
+ return uri;
+ }
+
/**
* Gets the service repos with duplicates filtered out. A service repo is considered duplicate if:
* <ul>
@@ -1233,7 +1277,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
if (null != serviceRepoXml) {
repos.addAll(serviceRepoXml.getRepositories());
if (null != serviceRepoXml.getLatestURI()) {
- stackContext.registerRepoUpdateTask(serviceRepoXml.getLatestURI(), this);
+ registerRepoUpdateTask(serviceRepoXml);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
index 4b79a71..477ee66 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackServiceDirectory.java
@@ -26,6 +26,7 @@ import javax.annotation.Nullable;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.state.stack.RepositoryXml;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,7 +101,11 @@ public class StackServiceDirectory extends ServiceDirectory {
String stackName = stackDir.getName();
String versionString = stackVersionDir.getName().replaceAll("\\.", "");
- return stackName + versionString + serviceName + "ServiceAdvisor";
+ // Remove illegal python characters from the advisor name
+ String advisorClassName = stackName + versionString + serviceName + "ServiceAdvisor";
+ advisorClassName = advisorClassName.replaceAll("[^a-zA-Z0-9]+", "");
+
+ return advisorClassName;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
index d57b5d6..8ab1fe9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryInfo.java
@@ -29,6 +29,8 @@ public class RepositoryInfo {
private String osType;
private String repoId;
private String repoName;
+ private String distribution;
+ private String components;
private String mirrorsList;
private String defaultBaseUrl;
private boolean repoSaved = false;
@@ -91,6 +93,22 @@ public class RepositoryInfo {
this.repoName = repoName;
}
+ public String getDistribution() {
+ return distribution;
+ }
+
+ public void setDistribution(String distribution) {
+ this.distribution = distribution;
+ }
+
+ public String getComponents() {
+ return components;
+ }
+
+ public void setComponents(String components) {
+ this.components = components;
+ }
+
/**
* @return the mirrorsList
*/
@@ -154,6 +172,8 @@ public class RepositoryInfo {
+ ", repoId=" + repoId
+ ", baseUrl=" + baseUrl
+ ", repoName=" + repoName
+ + ", distribution=" + distribution
+ + ", components=" + components
+ ", mirrorsList=" + mirrorsList
+ ", unique=" + unique
+ ", ambariManagedRepositories=" + ambariManagedRepositories
@@ -171,6 +191,8 @@ public class RepositoryInfo {
Objects.equal(osType, that.osType) &&
Objects.equal(repoId, that.repoId) &&
Objects.equal(repoName, that.repoName) &&
+ Objects.equal(distribution, that.distribution) &&
+ Objects.equal(components, that.components) &&
Objects.equal(mirrorsList, that.mirrorsList) &&
Objects.equal(defaultBaseUrl, that.defaultBaseUrl) &&
Objects.equal(ambariManagedRepositories, that.ambariManagedRepositories);
@@ -178,13 +200,14 @@ public class RepositoryInfo {
@Override
public int hashCode() {
- return Objects.hashCode(baseUrl, osType, repoId, repoName, mirrorsList, defaultBaseUrl, repoSaved, unique, ambariManagedRepositories);
+ return Objects.hashCode(baseUrl, osType, repoId, repoName, distribution, components, mirrorsList, defaultBaseUrl,
+ ambariManagedRepositories);
}
public RepositoryResponse convertToResponse()
{
return new RepositoryResponse(getBaseUrl(), getOsType(), getRepoId(),
- getRepoName(), getMirrorsList(), getDefaultBaseUrl());
+ getRepoName(), getDistribution(), getComponents(), getMirrorsList(), getDefaultBaseUrl());
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
index aecd55d..7efccd0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java
@@ -34,6 +34,7 @@ import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ObjectNotFoundException;
import org.apache.ambari.server.ServiceComponentHostNotFoundException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.MaintenanceStateHelper;
import org.apache.ambari.server.controller.ServiceComponentResponse;
import org.apache.ambari.server.controller.internal.DeleteHostComponentStatusMetaData;
import org.apache.ambari.server.events.ServiceComponentRecoveryChangedEvent;
@@ -101,6 +102,9 @@ public class ServiceComponentImpl implements ServiceComponent {
@Inject
private HostComponentStateDAO hostComponentDAO;
+ @Inject
+ private MaintenanceStateHelper maintenanceStateHelper;
+
@AssistedInject
public ServiceComponentImpl(@Assisted Service service, @Assisted String componentName,
AmbariMetaInfo ambariMetaInfo,
@@ -748,10 +752,31 @@ public class ServiceComponentImpl implements ServiceComponent {
return count;
}
+ /**
+ * Count the ServiceComponentHosts that have given state and are effectively not in maintenanceMode
+ * @param state
+ * @return
+ */
+ private int getMaintenanceOffSCHCountByState(State state) {
+ int count = 0;
+ for (ServiceComponentHost sch : hostComponents.values()) {
+ try {
+ MaintenanceState effectiveMaintenanceState = maintenanceStateHelper.getEffectiveState(sch, sch.getHost());
+ if (sch.getState() == state && effectiveMaintenanceState == MaintenanceState.OFF) {
+ count++;
+ }
+ } catch (AmbariException e) {
+ e.printStackTrace();
+ }
+ }
+ return count;
+ }
+
private Map <String, Integer> getServiceComponentStateCount() {
Map <String, Integer> serviceComponentStateCountMap = new HashMap<>();
serviceComponentStateCountMap.put("startedCount", getSCHCountByState(State.STARTED));
serviceComponentStateCountMap.put("installedCount", getSCHCountByState(State.INSTALLED));
+ serviceComponentStateCountMap.put("installedAndMaintenanceOffCount", getMaintenanceOffSCHCountByState(State.INSTALLED));
serviceComponentStateCountMap.put("installFailedCount", getSCHCountByState(State.INSTALL_FAILED));
serviceComponentStateCountMap.put("initCount", getSCHCountByState(State.INIT));
serviceComponentStateCountMap.put("unknownCount", getSCHCountByState(State.UNKNOWN));
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
index 50443dc..f692eb5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceOsSpecific.java
@@ -105,6 +105,10 @@ public class ServiceOsSpecific {
private String repoid;
@SerializedName("repoName")
private String reponame;
+ @SerializedName("distribution")
+ private String distribution;
+ @SerializedName("components")
+ private String components;
private Repo() {
}
@@ -137,6 +141,14 @@ public class ServiceOsSpecific {
return reponame;
}
+ public String getDistribution() {
+ return distribution;
+ }
+
+ public String getComponents() {
+ return components;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -148,6 +160,8 @@ public class ServiceOsSpecific {
if (mirrorslist != null ? !mirrorslist.equals(repo.mirrorslist) : repo.mirrorslist != null) return false;
if (repoid != null ? !repoid.equals(repo.repoid) : repo.repoid != null) return false;
if (reponame != null ? !reponame.equals(repo.reponame) : repo.reponame != null) return false;
+ if (distribution != null ? !distribution.equals(repo.distribution) : repo.distribution != null) return false;
+ if (components != null ? !components.equals(repo.components) : repo.components != null) return false;
return true;
}
@@ -158,6 +172,8 @@ public class ServiceOsSpecific {
result = 31 * result + (mirrorslist != null ? mirrorslist.hashCode() : 0);
result = 31 * result + (repoid != null ? repoid.hashCode() : 0);
result = 31 * result + (reponame != null ? reponame.hashCode() : 0);
+ result = 31 * result + (distribution != null ? distribution.hashCode() : 0);
+ result = 31 * result + (components != null ? components.hashCode() : 0);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index 6184b94..a3886ab 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@ -33,7 +33,6 @@ import org.apache.ambari.server.controller.StackVersionResponse;
import org.apache.ambari.server.stack.Validable;
import org.apache.ambari.server.state.repository.VersionDefinitionXml;
import org.apache.ambari.server.state.stack.ConfigUpgradePack;
-import org.apache.ambari.server.state.stack.LatestRepoCallable;
import org.apache.ambari.server.state.stack.RepositoryXml;
import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
import org.apache.ambari.server.state.stack.UpgradePack;
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
index 464cb41..8f9d8e1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java
@@ -563,15 +563,29 @@ public class UpgradeHelper {
LinkedHashSet<StageWrapper> priority = new LinkedHashSet<>();
LinkedHashSet<StageWrapper> others = new LinkedHashSet<>();
+ Set<String> extraKeys = new HashSet<>();
+ LinkedHashSet<StageWrapper> extras = new LinkedHashSet<>();
+
for (List<StageWrapper> holderItems : new List[] { oldHolder.items, newHolder.items }) {
for (StageWrapper stageWrapper : holderItems) {
- ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
-
- if (wrapper.priority) {
- priority.add(stageWrapper);
+ if (stageWrapper instanceof ServiceCheckStageWrapper) {
+ ServiceCheckStageWrapper wrapper = (ServiceCheckStageWrapper) stageWrapper;
+ if (wrapper.priority) {
+ priority.add(stageWrapper);
+ } else {
+ others.add(stageWrapper);
+ }
} else {
- others.add(stageWrapper);
+ // !!! It's a good chance that back-to-back service check groups are adding the
+ // same non-service-check wrappers.
+ // this should be "equal enough" to prevent them from duplicating on merge
+ String key = stageWrapper.toString();
+ if (!extraKeys.contains(key)) {
+ extras.add(stageWrapper);
+ extraKeys.add(key);
+ }
}
+
}
}
@@ -580,6 +594,7 @@ public class UpgradeHelper {
oldHolder.items = Lists.newLinkedList(priority);
oldHolder.items.addAll(others);
+ oldHolder.items.addAll(extras);
}
/**
@@ -911,7 +926,8 @@ public class UpgradeHelper {
* stack and the target stack. If a value has changed between stacks, then the
* target stack value should be taken unless the cluster's value differs from
* the old stack. This can occur if a property has been customized after
- * installation.</li>
+ * installation. Read-only properties, however, are always taken from the new
+ * stack.</li>
* <li>Downgrade: Reset the latest configurations from the service's original
* stack. The new configurations that were created on upgrade must be left
* intact until all components have been reverted, otherwise heartbeats will
@@ -961,6 +977,11 @@ public class UpgradeHelper {
continue;
}
+ // the auto-merge must take read-only properties even if they have changed
+ // - if the properties was read-only in the source stack, then we must
+ // take the new stack's value
+ Map<String, Set<String>> readOnlyProperties = getReadOnlyProperties(sourceStackId, serviceName);
+
// upgrade is a bit harder - we have to merge new stack configurations in
// populate a map of default configurations for the service on the old
@@ -1026,8 +1047,7 @@ public class UpgradeHelper {
Map<String, String> existingConfigurations = existingServiceConfig.getProperties();
// get the new configurations
- Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(
- configurationType);
+ Map<String, String> newDefaultConfigurations = newServiceDefaultConfigsByType.get(configurationType);
// if the new stack configurations don't have the type, then simply add
// all of the existing in
@@ -1046,8 +1066,7 @@ public class UpgradeHelper {
}
}
- // process every existing configuration property for this configuration
- // type
+ // process every existing configuration property for this configuration type
for (Map.Entry<String, String> existingConfigurationEntry : existingConfigurations.entrySet()) {
String existingConfigurationKey = existingConfigurationEntry.getKey();
String existingConfigurationValue = existingConfigurationEntry.getValue();
@@ -1064,17 +1083,22 @@ public class UpgradeHelper {
// from the original stack
String oldDefaultValue = oldServiceDefaultConfigs.get(existingConfigurationKey);
- if (!StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
- // at this point, we've determined that there is a
- // difference
- // between default values between stacks, but the value was
- // also customized, so keep the customized value
+ // see if this property is a read-only property which means that
+ // we shouldn't care if it was changed - we should take the new
+ // stack's value
+ Set<String> readOnlyPropertiesForType = readOnlyProperties.get(configurationType);
+ boolean readOnly = (null != readOnlyPropertiesForType
+ && readOnlyPropertiesForType.contains(existingConfigurationKey));
+
+ if (!readOnly && !StringUtils.equals(existingConfigurationValue, oldDefaultValue)) {
+ // at this point, we've determined that there is a difference
+ // between default values between stacks, but the value was also
+ // customized, so keep the customized value
newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
}
}
} else {
- // there is no entry in the map, so add the existing key/value
- // pair
+ // there is no entry in the map, so add the existing key/value pair
newDefaultConfigurations.put(existingConfigurationKey, existingConfigurationValue);
}
}
@@ -1128,4 +1152,55 @@ public class UpgradeHelper {
}
}
}
+
+ /**
+ * Gets all of the read-only properties for the given service. This will also
+ * include any stack properties as well which are read-only.
+ *
+ * @param stackId
+ * the stack to get read-only properties for (not {@code null}).
+ * @param serviceName
+ * the namee of the service (not {@code null}).
+ * @return a map of configuration type to set of property names which are
+ * read-only
+ * @throws AmbariException
+ */
+ private Map<String, Set<String>> getReadOnlyProperties(StackId stackId, String serviceName)
+ throws AmbariException {
+ Map<String, Set<String>> readOnlyProperties = new HashMap<>();
+
+ Set<PropertyInfo> properties = new HashSet<>();
+
+ Set<PropertyInfo> stackProperties = m_ambariMetaInfoProvider.get().getStackProperties(
+ stackId.getStackName(), stackId.getStackVersion());
+
+ Set<PropertyInfo> serviceProperties = m_ambariMetaInfoProvider.get().getServiceProperties(
+ stackId.getStackName(), stackId.getStackVersion(), serviceName);
+
+ if (CollectionUtils.isNotEmpty(stackProperties)) {
+ properties.addAll(stackProperties);
+ }
+
+ if (CollectionUtils.isNotEmpty(serviceProperties)) {
+ properties.addAll(serviceProperties);
+ }
+
+ for (PropertyInfo property : properties) {
+ ValueAttributesInfo valueAttributes = property.getPropertyValueAttributes();
+ if (null != valueAttributes && valueAttributes.getReadOnly() == Boolean.TRUE) {
+ String type = ConfigHelper.fileNameToConfigType(property.getFilename());
+
+ // get the set of properties for this type, initializing it if needed
+ Set<String> readOnlyPropertiesForType = readOnlyProperties.get(type);
+ if (null == readOnlyPropertiesForType) {
+ readOnlyPropertiesForType = new HashSet<>();
+ readOnlyProperties.put(type, readOnlyPropertiesForType);
+ }
+
+ readOnlyPropertiesForType.add(property.getName());
+ }
+ }
+
+ return readOnlyProperties;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 48d3f5b..bbbb613 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1974,6 +1974,8 @@ public class ClusterImpl implements Cluster {
}
}
+ clusterEntity = clusterDAO.merge(clusterEntity);
+
if (serviceName == null) {
ArrayList<String> configTypes = new ArrayList<>();
for (Config config: configs) {
@@ -2482,7 +2484,7 @@ public class ClusterImpl implements Cluster {
// since the entities which were modified came from the cluster entity's
// list to begin with, we can just save them right back - no need for a
// new collection since the entity instances were modified directly
- clusterEntity = clusterDAO.merge(clusterEntity);
+ clusterEntity = clusterDAO.merge(clusterEntity, true);
cacheConfigurations();
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
index 0d1da9c..b496942 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java
@@ -34,6 +34,8 @@ import org.apache.commons.lang.StringUtils;
*/
public abstract class AbstractKerberosDescriptor {
+ static final String KEY_NAME = "name";
+
/**
* An AbstractKerberosDescriptor serving as the parent (or container) for this
* AbstractKerberosDescriptor.
@@ -74,7 +76,7 @@ public abstract class AbstractKerberosDescriptor {
String name = getName();
if (name != null) {
- dataMap.put("name", name);
+ dataMap.put(KEY_NAME, name);
}
return dataMap;
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
index 5658133..4255dd1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java
@@ -96,6 +96,10 @@ import com.google.common.collect.Sets;
public abstract class AbstractKerberosDescriptorContainer extends AbstractKerberosDescriptor {
private static final Logger LOG = LoggerFactory.getLogger(AbstractKerberosDescriptorContainer.class);
+ static final String KEY_IDENTITIES = Type.IDENTITY.getDescriptorPluralName();
+ static final String KEY_CONFIGURATIONS = Type.CONFIGURATION.getDescriptorPluralName();
+ static final String KEY_AUTH_TO_LOCAL_PROPERTIES = Type.AUTH_TO_LOCAL_PROPERTY.getDescriptorPluralName();
+
/**
* Regular expression pattern used to parse auth_to_local property specifications into the following
* parts:
@@ -135,7 +139,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
Object list;
// (Safely) Get the set of KerberosIdentityDescriptors
- list = data.get(Type.IDENTITY.getDescriptorPluralName());
+ list = data.get(KEY_IDENTITIES);
if (list instanceof Collection) {
for (Object item : (Collection) list) {
if (item instanceof Map) {
@@ -145,7 +149,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
}
// (Safely) Get the set of KerberosConfigurationDescriptors
- list = data.get(Type.CONFIGURATION.getDescriptorPluralName());
+ list = data.get(KEY_CONFIGURATIONS);
if (list instanceof Collection) {
for (Object item : (Collection) list) {
if (item instanceof Map) {
@@ -155,7 +159,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
}
// (Safely) Get the set of KerberosConfigurationDescriptors
- list = data.get(Type.AUTH_TO_LOCAL_PROPERTY.getDescriptorPluralName());
+ list = data.get(KEY_AUTH_TO_LOCAL_PROPERTIES);
if (list instanceof Collection) {
for (Object item : (Collection) list) {
if (item instanceof String) {
@@ -636,7 +640,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
* @param path a String declaring the path to a KerberosIdentityDescriptor
* @return a KerberosIdentityDescriptor identified by the path or null if not found
*/
- protected KerberosIdentityDescriptor getReferencedIdentityDescriptor(String path)
+ public KerberosIdentityDescriptor getReferencedIdentityDescriptor(String path)
throws AmbariException {
KerberosIdentityDescriptor identityDescriptor = null;
@@ -759,7 +763,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
for (KerberosIdentityDescriptor identity : identities) {
list.put(identity.getName(), identity.toMap());
}
- map.put(Type.IDENTITY.getDescriptorPluralName(), list.values());
+ map.put(KEY_IDENTITIES, list.values());
}
if (configurations != null) {
@@ -769,11 +773,11 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
for (KerberosConfigurationDescriptor configuration : configurations.values()) {
list.put(configuration.getType(), configuration.toMap());
}
- map.put(Type.CONFIGURATION.getDescriptorPluralName(), list.values());
+ map.put(KEY_CONFIGURATIONS, list.values());
}
if (authToLocalProperties != null) {
- map.put(Type.AUTH_TO_LOCAL_PROPERTY.getDescriptorPluralName(), authToLocalProperties);
+ map.put(KEY_AUTH_TO_LOCAL_PROPERTIES, authToLocalProperties);
}
return map;
@@ -851,12 +855,7 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber
if (identity != null) {
KerberosIdentityDescriptor referencedIdentity;
try {
- if (identity.getReference() != null) {
- referencedIdentity = getReferencedIdentityDescriptor(identity.getReference());
- } else {
- // For backwards compatibility, see if the identity's name indicates a reference...
- referencedIdentity = getReferencedIdentityDescriptor(identity.getName());
- }
+ referencedIdentity = getReferencedIdentityDescriptor(identity.getReference());
} catch (AmbariException e) {
throw new AmbariException(String.format("Invalid Kerberos identity reference: %s", identity.getReference()), e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
index 768a17e..3bf1dad 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosComponentDescriptor.java
@@ -20,7 +20,7 @@ package org.apache.ambari.server.state.kerberos;
import java.util.Collection;
import java.util.Map;
-/**
+/*
* KerberosComponentDescriptor implements AbstractKerberosDescriptorContainer. It contains the data
* related to a component which include the following properties:
* <ul>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
index 9432f6c..0c7a9a9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosDescriptor.java
@@ -90,6 +90,9 @@ import org.apache.commons.lang.StringUtils;
*/
public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
+ static final String KEY_PROPERTIES = "properties";
+ static final String KEY_SERVICES = Type.SERVICE.getDescriptorPluralName();
+
/**
* A Map of the "global" properties contained within this KerberosDescriptor
*/
@@ -121,7 +124,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
super(data);
if (data != null) {
- Object list = data.get(Type.SERVICE.getDescriptorPluralName());
+ Object list = data.get(KEY_SERVICES);
if (list instanceof Collection) {
for (Object item : (Collection) list) {
if (item instanceof Map) {
@@ -130,7 +133,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
}
}
- Object map = data.get("properties");
+ Object map = data.get(KEY_PROPERTIES);
if (map instanceof Map) {
for (Map.Entry<?, ?> entry : ((Map<?, ?>) map).entrySet()) {
Object value = entry.getValue();
@@ -198,7 +201,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
}
if (services == null) {
- services = new TreeMap<>();
+ services = new TreeMap<String, KerberosServiceDescriptor>();
}
KerberosServiceDescriptor existing = services.get(name);
@@ -326,11 +329,11 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
for (KerberosServiceDescriptor service : services.values()) {
list.add(service.toMap());
}
- map.put(Type.SERVICE.getDescriptorPluralName(), list);
+ map.put(KEY_SERVICES, list);
}
if (properties != null) {
- map.put("properties", new TreeMap<>(properties));
+ map.put(KEY_PROPERTIES, new TreeMap<>(properties));
}
return map;
@@ -453,7 +456,7 @@ public class KerberosDescriptor extends AbstractKerberosDescriptorContainer {
private static void collectFromIdentities(String service, String component, Collection<KerberosIdentityDescriptor> identities, Map<String, String> result) {
for (KerberosIdentityDescriptor each : identities) {
- if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent() && !each.getName().startsWith("/")) {
+ if (each.getPrincipalDescriptor() != null && !each.getReferencedServiceName().isPresent()) {
String path = StringUtils.isBlank(component)
? String.format("%s/%s", service, each.getName())
: String.format("%s/%s/%s", service, component, each.getName());
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
index 911723b..ef45343 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java
@@ -33,7 +33,6 @@ import com.google.common.base.Optional;
* <li>name</li>
* <li>principal</li>
* <li>keytab</li>
- * <li>password</li>
* </ul>
* <p/>
* The following (pseudo) JSON Schema will yield a valid KerberosIdentityDescriptor
@@ -58,11 +57,6 @@ import com.google.common.base.Optional;
* "type": "{@link org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor}",
* }
* }
- * "password": {
- * "description": "The password to use for this identity. If not set a secure random
- * password will automatically be generated",
- * "type": "string"
- * }
* }
* }
* </pre>
@@ -73,6 +67,11 @@ import com.google.common.base.Optional;
*/
public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
+ static final String KEY_REFERENCE = "reference";
+ static final String KEY_PRINCIPAL = Type.PRINCIPAL.getDescriptorName();
+ static final String KEY_KEYTAB = Type.KEYTAB.getDescriptorName();
+ static final String KEY_WHEN = "when";
+
/**
* The path to the Kerberos Identity definitions this {@link KerberosIdentityDescriptor} references
*/
@@ -89,13 +88,6 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
private KerberosKeytabDescriptor keytab = null;
/**
- * A String containing the password for this Kerberos identity
- * <p/>
- * If this value is null or empty, a random password will be generated as necessary.
- */
- private String password = null;
-
- /**
* An expression used to determine when this {@link KerberosIdentityDescriptor} is relevant for the
* cluster. If the process expression is not <code>null</code> and evaluates to <code>false</code>
* then this {@link KerberosIdentityDescriptor} will be ignored when processing identities.
@@ -105,11 +97,11 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
/**
* Creates a new KerberosIdentityDescriptor
*
- * @param name the name of this identity descriptor
+ * @param name the name of this identity descriptor
* @param reference an optional path to a referenced KerberosIdentityDescriptor
* @param principal a KerberosPrincipalDescriptor
- * @param keytab a KerberosKeytabDescriptor
- * @param when a predicate
+ * @param keytab a KerberosKeytabDescriptor
+ * @param when a predicate
*/
public KerberosIdentityDescriptor(String name, String reference, KerberosPrincipalDescriptor principal, KerberosKeytabDescriptor keytab, Predicate when) {
setName(name);
@@ -133,24 +125,22 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
// This is not automatically set by the super classes.
setName(getStringValue(data, "name"));
- setReference(getStringValue(data, "reference"));
+ setReference(getStringValue(data, KEY_REFERENCE));
if (data != null) {
Object item;
- setPassword(getStringValue(data, "password"));
-
- item = data.get(Type.PRINCIPAL.getDescriptorName());
+ item = data.get(KEY_PRINCIPAL);
if (item instanceof Map) {
setPrincipalDescriptor(new KerberosPrincipalDescriptor((Map<?, ?>) item));
}
- item = data.get(Type.KEYTAB.getDescriptorName());
+ item = data.get(KEY_KEYTAB);
if (item instanceof Map) {
setKeytabDescriptor(new KerberosKeytabDescriptor((Map<?, ?>) item));
}
- item = data.get("when");
+ item = data.get(KEY_WHEN);
if (item instanceof Map) {
setWhen(PredicateUtils.fromMap((Map<?, ?>) item));
}
@@ -221,27 +211,6 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
}
/**
- * Gets the password for this this KerberosIdentityDescriptor
- *
- * @return A String containing the password for this this KerberosIdentityDescriptor
- * @see #password
- */
- public String getPassword() {
- return password;
- }
-
- /**
- * Sets the password for this this KerberosIdentityDescriptor
- *
- * @param password A String containing the password for this this KerberosIdentityDescriptor
- * @see #password
- */
- public void setPassword(String password) {
- this.password = password;
- }
-
-
- /**
* Gets the expression (or {@link Predicate}) to use to determine when to include this Kerberos
* identity while processing Kerberos identities.
* <p>
@@ -295,8 +264,6 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
setReference(updates.getReference());
- setPassword(updates.getPassword());
-
KerberosPrincipalDescriptor existingPrincipal = getPrincipalDescriptor();
if (existingPrincipal == null) {
setPrincipalDescriptor(updates.getPrincipalDescriptor());
@@ -312,7 +279,7 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
}
Predicate updatedWhen = updates.getWhen();
- if(updatedWhen != null) {
+ if (updatedWhen != null) {
setWhen(updatedWhen);
}
}
@@ -331,23 +298,19 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
Map<String, Object> dataMap = super.toMap();
if (reference != null) {
- dataMap.put("reference", reference);
+ dataMap.put(KEY_REFERENCE, reference);
}
if (principal != null) {
- dataMap.put(Type.PRINCIPAL.getDescriptorName(), principal.toMap());
+ dataMap.put(KEY_PRINCIPAL, principal.toMap());
}
if (keytab != null) {
- dataMap.put(Type.KEYTAB.getDescriptorName(), keytab.toMap());
- }
-
- if (password != null) {
- dataMap.put("password", password);
+ dataMap.put(KEY_KEYTAB, keytab.toMap());
}
- if(when != null) {
- dataMap.put("when", PredicateUtils.toMap(when));
+ if (when != null) {
+ dataMap.put(KEY_WHEN, PredicateUtils.toMap(when));
}
return dataMap;
@@ -435,11 +398,6 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor {
: getKeytabDescriptor().equals(descriptor.getKeytabDescriptor())
) &&
(
- (getPassword() == null)
- ? (descriptor.getPassword() == null)
- : getPassword().equals(descriptor.getPassword())
- ) &&
- (
(getWhen() == null)
? (descriptor.getWhen() == null)
: getWhen().equals(descriptor.getWhen())
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
index a17caad..7047c81 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosKeytabDescriptor.java
@@ -97,6 +97,14 @@ import java.util.TreeMap;
*/
public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
+ static final String KEY_FILE = "file";
+ static final String KEY_OWNER = "owner";
+ static final String KEY_GROUP = "group";
+ static final String KEY_CONFIGURATION = "configuration";
+ static final String KEY_CACHABLE = "cachable";
+ static final String KEY_ACL_NAME = "name";
+ static final String KEY_ACL_ACCESS = "access";
+
/**
* A String declaring the local username that should be set as the owner of the keytab file
*/
@@ -188,29 +196,29 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
public KerberosKeytabDescriptor(Map<?, ?> data) {
// The name for this KerberosKeytabDescriptor is stored in the "file" entry in the map
// This is not automatically set by the super classes.
- setName(getStringValue(data, "file"));
+ setName(getStringValue(data, KEY_FILE));
if (data != null) {
Object object;
- object = data.get("owner");
+ object = data.get(KEY_OWNER);
if (object instanceof Map) {
Map<?, ?> map = (Map<?, ?>) object;
- setOwnerName(getStringValue(map, "name"));
- setOwnerAccess(getStringValue(map, "access"));
+ setOwnerName(getStringValue(map, KEY_ACL_NAME));
+ setOwnerAccess(getStringValue(map, KEY_ACL_ACCESS));
}
- object = data.get("group");
+ object = data.get(KEY_GROUP);
if (object instanceof Map) {
Map<?, ?> map = (Map<?, ?>) object;
- setGroupName(getStringValue(map, "name"));
- setGroupAccess(getStringValue(map, "access"));
+ setGroupName(getStringValue(map, KEY_ACL_NAME));
+ setGroupAccess(getStringValue(map, KEY_ACL_ACCESS));
}
- setConfiguration(getStringValue(data, "configuration"));
+ setConfiguration(getStringValue(data, KEY_CONFIGURATION));
// If the "cachable" value is anything but false, set it to true
- setCachable(!"false".equalsIgnoreCase(getStringValue(data, "cachable")));
+ setCachable(!"false".equalsIgnoreCase(getStringValue(data, KEY_CACHABLE)));
}
}
@@ -422,23 +430,23 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
String data;
data = getFile();
- map.put("file", data);
+ map.put(KEY_FILE, data);
// Build file owner map
Map<String, String> owner = new TreeMap<>();
data = getOwnerName();
if (data != null) {
- owner.put("name", data);
+ owner.put(KEY_ACL_NAME, data);
}
data = getOwnerAccess();
if (data != null) {
- owner.put("access", data);
+ owner.put(KEY_ACL_ACCESS, data);
}
if (!owner.isEmpty()) {
- map.put("owner", owner);
+ map.put(KEY_OWNER, owner);
}
// Build file owner map (end)
@@ -447,22 +455,22 @@ public class KerberosKeytabDescriptor extends AbstractKerberosDescriptor {
data = getGroupName();
if (data != null) {
- group.put("name", data);
+ group.put(KEY_ACL_NAME, data);
}
data = getGroupAccess();
if (data != null) {
- group.put("access", data);
+ group.put(KEY_ACL_ACCESS, data);
}
if (!owner.isEmpty()) {
- map.put("group", group);
+ map.put(KEY_GROUP, group);
}
// Build file owner map (end)
data = getConfiguration();
if (data != null) {
- map.put("configuration", data);
+ map.put(KEY_CONFIGURATION, data);
}
return map;
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
index 74254e3..a8e094f94 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosPrincipalDescriptor.java
@@ -68,6 +68,11 @@ import java.util.TreeMap;
*/
public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
+ static final String KEY_VALUE = "value";
+ static final String KEY_TYPE = "type";
+ static final String KEY_CONFIGURATION = "configuration";
+ static final String KEY_LOCAL_USERNAME = "local_username";
+
/**
* A string declaring the type of principal this KerberosPrincipalDescriptor represents.
* <p/>
@@ -100,8 +105,8 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
/**
* Creates a new KerberosPrincipalDescriptor
*
- * @param principal the principal name
- * @param type the principal type (user, service, etc...)
+ * @param principal the principal name
+ * @param type the principal type (user, service, etc...)
* @param configuration the configuration used to store the principal name
* @param localUsername the local username to map to the principal
*/
@@ -124,10 +129,10 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
* @see org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor
*/
public KerberosPrincipalDescriptor(Map<?, ?> data) {
- this(getStringValue(data, "value"),
- getKerberosPrincipalTypeValue(data, "type"),
- getStringValue(data, "configuration"),
- getStringValue(data, "local_username")
+ this(getStringValue(data, KEY_VALUE),
+ getKerberosPrincipalTypeValue(data, KEY_TYPE),
+ getStringValue(data, KEY_CONFIGURATION),
+ getStringValue(data, KEY_LOCAL_USERNAME)
);
}
@@ -269,10 +274,10 @@ public class KerberosPrincipalDescriptor extends AbstractKerberosDescriptor {
public Map<String, Object> toMap() {
Map<String, Object> map = new TreeMap<>();
- map.put("value", getValue());
- map.put("type", KerberosPrincipalType.translate(getType()));
- map.put("configuration", getConfiguration());
- map.put("local_username", getLocalUsername());
+ map.put(KEY_VALUE, getValue());
+ map.put(KEY_TYPE, KerberosPrincipalType.translate(getType()));
+ map.put(KEY_CONFIGURATION, getConfiguration());
+ map.put(KEY_LOCAL_USERNAME, getLocalUsername());
return map;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
index 5da3399..51b7cd0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosServiceDescriptor.java
@@ -46,6 +46,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
* "title": "KerberosServiceDescriptor",
* "description": "Describes an Ambari service",
* "type": "object",
+ * "preconfigure": "boolean",
* "properties": {
* "name": {
* "description": "An identifying name for this service descriptor.",
@@ -85,6 +86,9 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
*/
public class KerberosServiceDescriptor extends AbstractKerberosDescriptorContainer {
+ static final String KEY_PRECONFIGURE = "preconfigure";
+ static final String KEY_COMPONENTS = Type.COMPONENT.getDescriptorPluralName();
+
/**
* A Map of the components contained within this KerberosServiceDescriptor
*/
@@ -137,7 +141,7 @@ public class KerberosServiceDescriptor extends AbstractKerberosDescriptorContain
setName(name);
if (data != null) {
- Object list = data.get(Type.COMPONENT.getDescriptorPluralName());
+ Object list = data.get(KEY_COMPONENTS);
if (list instanceof Collection) {
// Assume list is Collection<Map<String, Object>>
for (Object item : (Collection) list) {
@@ -147,7 +151,7 @@ public class KerberosServiceDescriptor extends AbstractKerberosDescriptorContain
}
}
- setPreconfigure(getBooleanValue(data, "preconfigure"));
+ setPreconfigure(getBooleanValue(data, KEY_PRECONFIGURE));
}
}
@@ -275,11 +279,11 @@ public class KerberosServiceDescriptor extends AbstractKerberosDescriptorContain
for (KerberosComponentDescriptor component : components.values()) {
list.add(component.toMap());
}
- map.put(Type.COMPONENT.getDescriptorPluralName(), list);
+ map.put(KEY_COMPONENTS, list);
}
if (preconfigure != null) {
- map.put("preProcess", preconfigure.toString());
+ map.put(KEY_PRECONFIGURE, preconfigure.toString());
}
return map;
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
index b8058c8..d3237a9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AmbariServerAlertService.java
@@ -30,7 +30,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.ambari.server.AmbariService;
import org.apache.ambari.server.alerts.AlertRunnable;
import org.apache.ambari.server.controller.RootServiceResponseFactory.Components;
-import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
import org.apache.ambari.server.state.Cluster;
@@ -113,28 +112,18 @@ public class AmbariServerAlertService extends AbstractScheduledService {
/**
* {@inheritDoc}
* <p/>
- * Loads all of the {@link Components#AMBARI_SERVER} definitions and schedules
+ * Loads all of the definitions with SERVER source type and schedules
* the ones that are enabled.
*/
@Override
protected void startUp() throws Exception {
Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
for (Cluster cluster : clusterMap.values()) {
- List<AlertDefinitionEntity> entities = m_dao.findByServiceComponent(
- cluster.getClusterId(), Services.AMBARI.name(),
- Components.AMBARI_SERVER.name());
-
- for (AlertDefinitionEntity entity : entities) {
+ for (AlertDefinitionEntity entity : m_dao.findBySourceType(cluster.getClusterId(), SourceType.SERVER)) {
// don't schedule disabled alert definitions
if (!entity.getEnabled()) {
continue;
}
-
- SourceType sourceType = entity.getSourceType();
- if (sourceType != SourceType.SERVER) {
- continue;
- }
-
// schedule the Runnable for the definition
scheduleRunnable(entity);
}
@@ -152,10 +141,8 @@ public class AmbariServerAlertService extends AbstractScheduledService {
protected void runOneIteration() throws Exception {
Map<String, Cluster> clusterMap = m_clustersProvider.get().getClusters();
for (Cluster cluster : clusterMap.values()) {
- // get all of the cluster alerts for AMBARI/AMBARI_SERVER
- List<AlertDefinitionEntity> entities = m_dao.findByServiceComponent(
- cluster.getClusterId(), Services.AMBARI.name(),
- Components.AMBARI_SERVER.name());
+ // get all of the cluster alerts with SERVER source type
+ List<AlertDefinitionEntity> entities = m_dao.findBySourceType(cluster.getClusterId(), SourceType.SERVER);
// for each alert, check to see if it's scheduled correctly
for (AlertDefinitionEntity entity : entities) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoUrlInfoCallable.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoUrlInfoCallable.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoUrlInfoCallable.java
new file mode 100644
index 0000000..ec1071e
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoUrlInfoCallable.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.ambari.server.controller.internal.URLStreamProvider;
+import org.apache.ambari.server.stack.StackModule;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.stack.RepoUrlInfoCallable.RepoUrlInfoResult;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+/**
+ * Encapsulates the work to resolve the latest repo information for a stack.
+ * This class must be used AFTER the stack has created its owned repositories.
+ */
+public class RepoUrlInfoCallable implements Callable<Map<StackModule, RepoUrlInfoResult>> {
+ private static final int LOOKUP_CONNECTION_TIMEOUT = 2000;
+ private static final int LOOKUP_READ_TIMEOUT = 3000;
+
+ private final static Logger LOG = LoggerFactory.getLogger(RepoUrlInfoCallable.class);
+
+ private URI m_uri = null;
+ private Set<StackModule> m_stacks = new HashSet<>();
+
+ public RepoUrlInfoCallable(URI uri) {
+ m_uri = uri;
+ }
+
+ public void addStack(StackModule stackModule) {
+ m_stacks.add(stackModule);
+ }
+
+ @Override
+ public Map<StackModule, RepoUrlInfoResult> call() throws Exception {
+
+ Type type = new TypeToken<Map<String, Map<String, Object>>>(){}.getType();
+ Gson gson = new Gson();
+
+ Map<String, Map<String, Object>> latestUrlMap = null;
+
+ Set<String> ids = new HashSet<>();
+ ids.addAll(Collections2.transform(m_stacks, new Function<StackModule, String>() {
+ @Override
+ public String apply(StackModule input) {
+ // TODO Auto-generated method stub
+ return new StackId(input.getModuleInfo()).toString();
+ }
+ }));
+
+ String stackIds = StringUtils.join(ids, ',');
+
+ Long time = System.nanoTime();
+
+ try {
+ if (m_uri.getScheme().startsWith("http")) {
+ URLStreamProvider streamProvider = new URLStreamProvider(
+ LOOKUP_CONNECTION_TIMEOUT, LOOKUP_READ_TIMEOUT,
+ null, null, null);
+
+ LOG.info("Loading latest URL info from {} for stacks {}", m_uri, stackIds);
+
+ latestUrlMap = gson.fromJson(new InputStreamReader(
+ streamProvider.readFrom(m_uri.toString())), type);
+ } else {
+ File jsonFile = new File(m_uri);
+
+ if (jsonFile.exists()) {
+ LOG.info("Loading latest URL info from file {} for stacks {}", m_uri, stackIds);
+ latestUrlMap = gson.fromJson(new FileReader(jsonFile), type);
+ }
+ }
+ } catch (Exception e) {
+ LOG.info("Could not load the URI from {}, stack defaults will be used", m_uri);
+ throw e;
+ } finally {
+ LOG.info("Loaded URI {} for stacks {} in {}ms", m_uri, stackIds,
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - time));
+ }
+
+ Map<StackModule, RepoUrlInfoResult> result = new HashMap<>();
+
+ if (null == latestUrlMap) {
+ LOG.error("Could not load latest data for URI {} and stacks {}", m_uri, stackIds);
+ return result;
+ }
+
+ for (StackModule stackModule : m_stacks) {
+ StackId stackId = new StackId(stackModule.getModuleInfo());
+
+ Map<String, Object> map = latestUrlMap.get(stackId.toString());
+
+ if (null == map) {
+ continue;
+ }
+
+ RepoUrlInfoResult res = new RepoUrlInfoResult();
+
+ if (map.containsKey("manifests")) {
+ // versionMap is laid out like [version -> [os -> VDF uri]]
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, String>> versionMap = (Map<String, Map<String, String>>) map.get("manifests");
+
+ for (Entry<String, Map<String, String>> versionEntry : versionMap.entrySet()) {
+ String version = versionEntry.getKey();
+ Map<String, URI> resolvedOsMap = resolveOsMap(stackModule, versionEntry.getValue());
+
+ res.addVersion(version, resolvedOsMap);
+ }
+ }
+
+
+ if (map.containsKey("latest-vdf")) {
+ @SuppressWarnings("unchecked")
+ Map<String, String> osMap = (Map<String, String>) map.get("latest-vdf");
+
+ Map<String, URI> resolvedOsMap = resolveOsMap(stackModule, osMap);
+
+ res.setLatest(resolvedOsMap);
+ }
+
+ result.put(stackModule, res);
+ }
+
+
+ return result;
+
+ }
+
+ private Map<String, URI> resolveOsMap(StackModule stackModule, Map<String, String> osMap) {
+
+ Map<String, URI> resolved = new HashMap<>();
+
+ for (Entry<String, String> osEntry : osMap.entrySet()) {
+
+ String uriString = osEntry.getValue();
+
+ URI uri = StackModule.getURI(stackModule, uriString);
+
+ if (null == uri) {
+ LOG.warn("Could not resolve URI {}", uriString);
+ } else {
+ resolved.put(osEntry.getKey(), uri);
+ }
+ }
+
+ return resolved;
+ }
+
+
+ /**
+ * Stores the results saved per StackModule
+ */
+ public static class RepoUrlInfoResult {
+
+ private Map<String, Map<String, URI>> versions = new HashMap<>();
+ private Map<String, URI> latestVdf = new HashMap<>();
+
+ private void addVersion(String version, Map<String, URI> vdfMap) {
+ versions.put(version, vdfMap);
+ }
+
+ private void setLatest(Map<String, URI> latestMap) {
+ latestVdf = latestMap;
+ }
+
+ /**
+ * Each version entry here should be loaded in it's entirety in a new thread
+ */
+ public Map<String, Map<String, URI>> getManifest() {
+
+ return versions;
+ }
+
+ /**
+ * @return the latest vdf map
+ */
+ public Map<String, URI> getLatestVdf() {
+
+ return latestVdf;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoVdfCallable.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoVdfCallable.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoVdfCallable.java
new file mode 100644
index 0000000..81657a7
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepoVdfCallable.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import java.net.URI;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import org.apache.ambari.server.stack.StackModule;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Encapsulates the work to resolve the latest repo information for a stack.
+ * This class must be used AFTER the stack has created its owned repositories.
+ */
+public class RepoVdfCallable implements Callable<Void> {
+
+ private final static Logger LOG = LoggerFactory.getLogger(RepoVdfCallable.class);
+
+ // !!! these are required for this callable to work
+ private final StackInfo m_stack;
+ private final OsFamily m_family;
+ private final Map<String, URI> m_vdfMap;
+
+ // !!! determines if this is for manifests or latest-vdf
+ private String m_version;
+
+ public RepoVdfCallable(StackModule stackModule,
+ String version, Map<String, URI> vdfOsMap, OsFamily os_family) {
+ m_stack = stackModule.getModuleInfo();
+ m_family = os_family;
+ m_version = version;
+ m_vdfMap = vdfOsMap;
+ }
+
+ public RepoVdfCallable(StackModule stackModule,
+ Map<String, URI> vdfOsMap, OsFamily os_family) {
+ m_stack = stackModule.getModuleInfo();
+ m_family = os_family;
+ m_version = null;
+ m_vdfMap = vdfOsMap;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ if (MapUtils.isEmpty(m_vdfMap)) {
+ return null;
+ }
+
+ boolean forLatest = (null == m_version);
+
+ StackId stackId = new StackId(m_stack);
+
+ VersionDefinitionXml xml = mergeDefinitions(stackId, m_version, m_vdfMap);
+
+ if (null == xml) {
+ return null;
+ }
+
+ if (forLatest) {
+ xml.setStackDefault(true);
+ m_stack.setLatestVersionDefinition(xml);
+ } else {
+ m_stack.addVersionDefinition(m_version, xml);
+ }
+
+ return null;
+ }
+
+ /**
+ * Merges definitions loaded from the common file
+ * @param stackId the stack id
+ * @param version the version string
+ * @param osMap the map containing all the VDF for an OS
+ * @return the merged version definition
+ * @throws Exception
+ */
+ private VersionDefinitionXml mergeDefinitions(StackId stackId, String version,
+ Map<String, URI> osMap) throws Exception {
+
+ Set<String> oses = new HashSet<>();
+ for (RepositoryInfo ri : m_stack.getRepositories()) {
+ if (null != m_family.find(ri.getOsType())) {
+ oses.add(m_family.find(ri.getOsType()));
+ }
+ }
+
+ VersionDefinitionXml.Merger merger = new VersionDefinitionXml.Merger();
+
+ for (Entry<String, URI> versionEntry : osMap.entrySet()) {
+
+ String osFamily = m_family.find(versionEntry.getKey());
+ URI uri = versionEntry.getValue();
+
+ // !!! check for aliases. Moving this to OsFamily could result in incorrect behavior
+ if (null == osFamily) {
+ String alias = m_family.getAliases().get(versionEntry.getKey());
+ if (null != alias) {
+ osFamily = m_family.find(alias);
+ }
+ }
+
+ // !!! if the family is not known OR not part of the stack, skip
+ if (null == osFamily || !oses.contains(osFamily)) {
+ LOG.info("Stack {} cannot resolve OS {} to the supported ones: {}. Family: {}",
+ stackId, versionEntry.getKey(), StringUtils.join(oses, ','), osFamily);
+ continue;
+ }
+
+ try {
+ VersionDefinitionXml xml = timedVDFLoad(uri);
+
+ version = (null == version) ? xml.release.version : version;
+ merger.add(version, xml);
+
+ } catch (Exception e) {
+ LOG.warn("Could not load version definition for {} identified by {}. {}",
+ stackId, uri.toString(), e.getMessage(), e);
+ }
+ }
+
+ return merger.merge();
+ }
+
+ private VersionDefinitionXml timedVDFLoad(URI uri) throws Exception {
+ long time = System.currentTimeMillis();
+
+ try {
+ return VersionDefinitionXml.load(uri.toURL());
+ } finally {
+ LOG.debug("Loaded VDF {} in {}ms", uri, System.currentTimeMillis() - time);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
index 03b3705..c2209bb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java
@@ -146,6 +146,8 @@ public class RepositoryXml implements Validable{
private String mirrorslist = null;
private String repoid = null;
private String reponame = null;
+ private String distribution = null;
+ private String components = null;
private boolean unique = false;
private Repo() {
@@ -179,6 +181,13 @@ public class RepositoryXml implements Validable{
return reponame;
}
+ public String getDistribution() {
+ return distribution;
+ }
+
+ public String getComponents() {
+ return components;
+ }
/**
* @return true if version of HDP that change with each release
*/
@@ -212,6 +221,8 @@ public class RepositoryXml implements Validable{
ri.setOsType(os.trim());
ri.setRepoId(r.getRepoId());
ri.setRepoName(r.getRepoName());
+ ri.setDistribution(r.getDistribution());
+ ri.setComponents(r.getComponents());
ri.setUnique(r.isUnique());
repos.add(ri);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index 68dc63f..f88691d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -110,6 +110,9 @@ public class ConfigureTask extends ServerSideActionTask {
@XmlAttribute(name = "id")
public String id;
+ @XmlAttribute(name="supports-patch")
+ public boolean supportsPatch = false;
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
index 9524c09..f540d8d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java
@@ -124,6 +124,12 @@ public class RepositoryVersionHelper {
repositoryEntity.setBaseUrl(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID).getAsString());
repositoryEntity.setName(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID).getAsString());
repositoryEntity.setRepositoryId(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID).getAsString());
+ if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_DISTRIBUTION_PROPERTY_ID) != null) {
+ repositoryEntity.setDistribution(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_DISTRIBUTION_PROPERTY_ID).getAsString());
+ }
+ if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_COMPONENTS_PROPERTY_ID) != null) {
+ repositoryEntity.setComponents(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_COMPONENTS_PROPERTY_ID).getAsString());
+ }
if (repositoryJson.get(RepositoryResourceProvider.REPOSITORY_MIRRORS_LIST_PROPERTY_ID) != null) {
repositoryEntity.setMirrorsList(repositoryJson.get(RepositoryResourceProvider.REPOSITORY_MIRRORS_LIST_PROPERTY_ID).getAsString());
}
@@ -177,6 +183,8 @@ public class RepositoryVersionHelper {
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_BASE_URL_PROPERTY_ID, repository.getBaseUrl());
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_REPO_NAME_PROPERTY_ID, repository.getRepoName());
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_REPO_ID_PROPERTY_ID, repository.getRepoId());
+ repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_DISTRIBUTION_PROPERTY_ID, repository.getDistribution());
+ repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_COMPONENTS_PROPERTY_ID, repository.getComponents());
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_MIRRORS_LIST_PROPERTY_ID, repository.getMirrorsList());
repositoryJson.addProperty(RepositoryResourceProvider.REPOSITORY_UNIQUE_PROPERTY_ID, repository.isUnique());
repositoriesJson.add(repositoryJson);
@@ -196,6 +204,8 @@ public class RepositoryVersionHelper {
RepositoryInfo repositoryInfo = new RepositoryInfo();
repositoryInfo.setRepoId(repositoryEntity.getRepositoryId());
repositoryInfo.setRepoName(repositoryEntity.getName());
+ repositoryInfo.setDistribution(repositoryEntity.getDistribution());
+ repositoryInfo.setComponents(repositoryEntity.getComponents());
repositoryInfo.setBaseUrl(repositoryEntity.getBaseUrl());
repositoryInfo.setOsType(os.getOsType());
repositoryInfo.setAmbariManagedRepositories(os.isAmbariManagedRepos());
@@ -310,6 +320,7 @@ public class RepositoryVersionHelper {
commandRepo.setRepositories(osEntity.getOsType(), osEntity.getRepositories());
commandRepo.setRepositoryVersion(repoVersion.getVersion());
commandRepo.setRepositoryVersionId(repoVersion.getId());
+ commandRepo.setResolved(repoVersion.isResolved());
commandRepo.setStackName(repoVersion.getStackId().getStackName());
if (!osEntity.isAmbariManagedRepos()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 67f23ac..3e894a8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -206,7 +206,7 @@ public class AmbariContext {
StackId stackId = new StackId(stack.getName(), stack.getVersion());
RepositoryVersionEntity repoVersion = null;
- if (null == repoVersionString && null == repoVersionId) {
+ if (StringUtils.isEmpty(repoVersionString) && null == repoVersionId) {
List<RepositoryVersionEntity> stackRepoVersions = repositoryVersionDAO.findByStack(stackId);
if (stackRepoVersions.isEmpty()) {
@@ -769,8 +769,8 @@ public class AmbariContext {
}
});
- ConfigGroupRequest request = new ConfigGroupRequest(
- null, clusterName, absoluteGroupName, service, "Host Group Configuration",
+ ConfigGroupRequest request = new ConfigGroupRequest(null, clusterName,
+ absoluteGroupName, service, service, "Host Group Configuration",
Sets.newHashSet(filteredGroupHosts), serviceConfigs);
// get the config group provider and create config group resource
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
index 0863e37..9769fae 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java
@@ -29,9 +29,10 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
-import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
@@ -73,13 +74,13 @@ import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTask;
import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory;
import org.apache.ambari.server.topology.validators.TopologyValidatorService;
+import org.apache.ambari.server.utils.ManagedThreadPoolExecutor;
import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.eventbus.Subscribe;
import com.google.inject.Inject;
-import com.google.inject.Injector;
import com.google.inject.Singleton;
import com.google.inject.persist.Transactional;
@@ -103,9 +104,23 @@ public class TopologyManager {
private static final String CLUSTER_CONFIG_TASK_MAX_TIME_IN_MILLIS_PROPERTY_NAME = "cluster_configure_task_timeout";
private PersistedState persistedState;
+
+ /**
+ * Single threaded executor to execute async tasks. At the moment it's only used to execute ConfigureClusterTask.
+ */
private final ExecutorService executor = Executors.newSingleThreadExecutor();
- private final Executor taskExecutor; // executes TopologyTasks
- private final boolean parallelTaskCreationEnabled;
+
+ /**
+ * Thread pool size for topology task executors.
+ */
+ private int topologyTaskExecutorThreadPoolSize;
+ /**
+ * There is one ExecutorService for each cluster to execute TopologyTasks.
+ * TopologyTasks are submitted into ExecutorService for each cluster,
+ * however the ExecutorService is started only after cluster configuration is finished.
+ */
+ private final Map<Long, ManagedThreadPoolExecutor> topologyTaskExecutorServiceMap = new HashMap<>();
+
private Collection<String> hostsToIgnore = new HashSet<>();
private final List<HostImpl> availableHosts = new LinkedList<>();
private final Map<String, LogicalRequest> reservedHosts = new HashMap<>();
@@ -116,9 +131,6 @@ public class TopologyManager {
private Map<Long, ClusterTopology> clusterTopologyMap = new HashMap<>();
@Inject
- private Injector injector;
-
- @Inject
private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor;
@Inject
@@ -162,17 +174,15 @@ public class TopologyManager {
private Map<Long, Boolean> clusterProvisionWithBlueprintCreationFinished = new HashMap<>();
public TopologyManager() {
- parallelTaskCreationEnabled = false;
- taskExecutor = executor;
+ topologyTaskExecutorThreadPoolSize = 1;
}
@Inject
public TopologyManager(Configuration configuration) {
- int threadPoolSize = configuration.getParallelTopologyTaskCreationThreadPoolSize();
- parallelTaskCreationEnabled = configuration.isParallelTopologyTaskCreationEnabled() && threadPoolSize > 1;
- taskExecutor = parallelTaskCreationEnabled
- ? Executors.newFixedThreadPool(threadPoolSize)
- : executor;
+ topologyTaskExecutorThreadPoolSize = configuration.getParallelTopologyTaskCreationThreadPoolSize();
+ if (!configuration.isParallelTopologyTaskCreationEnabled()) {
+ topologyTaskExecutorThreadPoolSize = 1;
+ }
}
// executed by the IoC framework after creating the object (guice)
@@ -277,10 +287,6 @@ public class TopologyManager {
SecurityType securityType = null;
Credential credential = null;
- if (null == repoVersion && null == repoVersionID) {
- throw new AmbariException("Repository should be created and the version passed in the request.");
- }
-
SecurityConfiguration securityConfiguration = processSecurityConfiguration(request);
if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS) {
@@ -318,6 +324,10 @@ public class TopologyManager {
// set provision action requested
topology.setProvisionAction(request.getProvisionAction());
+
+ // create task executor for TopologyTasks
+ getOrCreateTopologyTaskExecutor(clusterId);
+
// persist request
LogicalRequest logicalRequest = RetryHelper.executeWithRetry(new Callable<LogicalRequest>() {
@Override
@@ -333,15 +343,6 @@ public class TopologyManager {
addClusterConfigRequest(topology, new ClusterConfigurationRequest(ambariContext, topology, true,
stackAdvisorBlueprintProcessor, securityType == SecurityType.KERBEROS));
- // Notify listeners that cluster configuration finished
- executor.submit(new Callable<Boolean>() {
- @Override
- public Boolean call() throws Exception {
- ambariEventPublisher.publish(new ClusterConfigFinishedEvent(clusterName));
- return Boolean.TRUE;
- }
- });
-
// Process the logical request
processRequest(request, topology, logicalRequest);
@@ -353,6 +354,17 @@ public class TopologyManager {
return getRequestStatus(logicalRequest.getRequestId());
}
+ @Subscribe
+ public void onClusterConfigFinishedEvent(ClusterConfigFinishedEvent event) {
+ ManagedThreadPoolExecutor taskExecutor = topologyTaskExecutorServiceMap.get(event.getClusterId());
+ if (taskExecutor == null) {
+ LOG.error("Can't find executor service taskQueue not found for cluster: {} ", event.getClusterName());
+ } else {
+ LOG.info("Starting topology task ExecutorService for cluster: {}", event.getClusterName());
+ taskExecutor.start();
+ }
+ }
+
/**
* Saves the quick links profile to the DB as an Ambari setting. Creates a new setting entity or updates the existing
@@ -949,16 +961,8 @@ public class TopologyManager {
}
LOG.info("TopologyManager.processAcceptedHostOffer: queue tasks for host = {} which responded {}", hostName, response.getAnswer());
- if (parallelTaskCreationEnabled) {
- executor.execute(new Runnable() { // do not start until cluster config done
- @Override
- public void run() {
- queueHostTasks(topology, response, hostName);
- }
- });
- } else {
- queueHostTasks(topology, response, hostName);
- }
+ queueHostTasks(topology, response, hostName);
+
}
@Transactional
@@ -967,9 +971,23 @@ public class TopologyManager {
persistedState.registerInTopologyHostInfo(host);
}
+ private ExecutorService getOrCreateTopologyTaskExecutor(Long clusterId) {
+ ManagedThreadPoolExecutor topologyTaskExecutor = this.topologyTaskExecutorServiceMap.get(clusterId);
+ if (topologyTaskExecutor == null) {
+ LOG.info("Creating TopologyTaskExecutorService for clusterId: {}", clusterId);
+
+ topologyTaskExecutor = new ManagedThreadPoolExecutor(topologyTaskExecutorThreadPoolSize,
+ topologyTaskExecutorThreadPoolSize, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>());
+ topologyTaskExecutorServiceMap.put(clusterId, topologyTaskExecutor);
+ }
+ return topologyTaskExecutor;
+ }
+
private void queueHostTasks(ClusterTopology topology, HostOfferResponse response, String hostName) {
LOG.info("TopologyManager.processAcceptedHostOffer: queueing tasks for host = {}", hostName);
- response.executeTasks(taskExecutor, hostName, topology, ambariContext);
+ ExecutorService executorService = getOrCreateTopologyTaskExecutor(topology.getClusterId());
+ response.executeTasks(executorService, hostName, topology, ambariContext);
}
private void updateHostWithRackInfo(ClusterTopology topology, HostOfferResponse response, HostImpl host) {
@@ -1116,7 +1134,7 @@ public class TopologyManager {
}
ConfigureClusterTask configureClusterTask = configureClusterTaskFactory.createConfigureClusterTask(topology,
- configurationRequest);
+ configurationRequest, ambariEventPublisher);
AsyncCallableService<Boolean> asyncCallableService = new AsyncCallableService<>(configureClusterTask, timeout, delay,
Executors.newScheduledThreadPool(1));
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
index 0ce5982..60eaa59 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTask.java
@@ -22,6 +22,8 @@ import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
+import org.apache.ambari.server.events.ClusterConfigFinishedEvent;
+import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.security.authorization.internal.RunWithInternalSecurityContext;
import org.apache.ambari.server.topology.ClusterConfigurationRequest;
import org.apache.ambari.server.topology.ClusterTopology;
@@ -39,11 +41,14 @@ public class ConfigureClusterTask implements Callable<Boolean> {
private ClusterConfigurationRequest configRequest;
private ClusterTopology topology;
+ private AmbariEventPublisher ambariEventPublisher;
@AssistedInject
- public ConfigureClusterTask(@Assisted ClusterTopology topology, @Assisted ClusterConfigurationRequest configRequest) {
+ public ConfigureClusterTask(@Assisted ClusterTopology topology, @Assisted ClusterConfigurationRequest configRequest,
+ @Assisted AmbariEventPublisher ambariEventPublisher) {
this.configRequest = configRequest;
this.topology = topology;
+ this.ambariEventPublisher = ambariEventPublisher;
}
@Override
@@ -72,6 +77,12 @@ public class ConfigureClusterTask implements Callable<Boolean> {
throw new Exception(e);
}
+ LOG.info("Cluster configuration finished successfully!");
+ // Notify listeners that cluster configuration finished
+ long clusterId = topology.getClusterId();
+ ambariEventPublisher.publish(new ClusterConfigFinishedEvent(clusterId,
+ topology.getAmbariContext().getClusterName(clusterId)));
+
LOG.info("TopologyManager.ConfigureClusterTask: Exiting");
return true;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5ae98dbe/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTaskFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTaskFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTaskFactory.java
index 9e3c151..558af30 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTaskFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/ConfigureClusterTaskFactory.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.topology.tasks;
+import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
import org.apache.ambari.server.topology.ClusterConfigurationRequest;
import org.apache.ambari.server.topology.ClusterTopology;
@@ -25,6 +26,6 @@ import org.apache.ambari.server.topology.ClusterTopology;
public interface ConfigureClusterTaskFactory {
ConfigureClusterTask createConfigureClusterTask(ClusterTopology topology, ClusterConfigurationRequest
- configRequest);
+ configRequest, AmbariEventPublisher ambariEventPublisher);
}