You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/04/22 07:39:39 UTC
[17/50] [abbrv] hadoop git commit: YARN-5993. Allow native services
quicklinks to be exported for each component. Contributed by Billie Rinaldi
YARN-5993. Allow native services quicklinks to be exported for each component. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcf0ff62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcf0ff62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcf0ff62
Branch: refs/heads/yarn-native-services
Commit: fcf0ff6256bd34fb2f63533d75dda7cca3aa929c
Parents: 3925462
Author: Gour Saha <go...@apache.org>
Authored: Wed Dec 28 15:24:36 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Sat Apr 22 15:31:50 2017 +0800
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 8 +-
.../core/registry/docstore/ExportEntry.java | 20 +++
.../registry/docstore/PublishedExports.java | 23 ++-
.../apache/slider/providers/ProviderUtils.java | 133 ++++------------
.../providers/docker/DockerProviderService.java | 151 ++++++++++++-------
.../server/appmaster/web/view/IndexBlock.java | 15 +-
6 files changed, 177 insertions(+), 173 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a107dfb..20c7831 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -183,17 +183,17 @@ public final class SliderUtils {
return !isUnset(s);
}
- public static boolean isEmpty(List l) {
+ public static boolean isEmpty(Collection l) {
return l == null || l.isEmpty();
}
/**
- * Probe for a list existing and not being empty
- * @param l list
+ * Probe for a collection existing and not being empty
+ * @param l collection
* @return true if the reference is valid and it contains entries
*/
- public static boolean isNotEmpty(List l) {
+ public static boolean isNotEmpty(Collection l) {
return l != null && !l.isEmpty();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
index 4bcf6c1..dd6e034 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -107,6 +107,26 @@ public class ExportEntry {
}
@Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ExportEntry that = (ExportEntry) o;
+
+ if (value != null ? !value.equals(that.value) : that.value != null)
+ return false;
+ return containerId != null ? containerId.equals(that.containerId) :
+ that.containerId == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = value != null ? value.hashCode() : 0;
+ result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
+ return result;
+ }
+
+ @Override
public String toString() {
return new StringBuilder("ExportEntry{").
append("value='").append(value).append("',").
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
index 0759b4e..58e67ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -26,8 +26,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
/**
* JSON-serializable description of a published key-val configuration.
@@ -41,7 +43,7 @@ public class PublishedExports {
public String description;
public long updated;
public String updatedTime;
- public Map<String, List<ExportEntry>> entries = new HashMap<>();
+ public Map<String, Set<ExportEntry>> entries = new HashMap<>();
public PublishedExports() {
}
@@ -62,7 +64,7 @@ public class PublishedExports {
* @param entries entries to put
*/
public PublishedExports(String description,
- Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ Iterable<Entry<String, Set<ExportEntry>>> entries) {
this.description = description;
putValues(entries);
}
@@ -87,15 +89,22 @@ public class PublishedExports {
this.updatedTime = new Date(updated).toString();
}
+
+ public Map<String, Set<ExportEntry>> sortedEntries() {
+ Map<String, Set<ExportEntry>> sortedEntries = new TreeMap<>();
+ sortedEntries.putAll(entries);
+ return sortedEntries;
+ }
+
/**
* Set the values from an iterable (this includes a Hadoop Configuration and Java properties
* object). Any existing value set is discarded
*
- * @param entries entries to put
+ * @param values values to put
*/
- public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
- this.entries = new HashMap<String, List<ExportEntry>>();
- for (Map.Entry<String, List<ExportEntry>> entry : entries) {
+ public void putValues(Iterable<Map.Entry<String, Set<ExportEntry>>> values) {
+ this.entries = new HashMap<>();
+ for (Map.Entry<String, Set<ExportEntry>> entry : values) {
this.entries.put(entry.getKey(), entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 0d9ddec..f33db9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -75,6 +75,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.regex.Pattern;
/**
@@ -1054,32 +1055,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
- * Return a list of hostnames based on current ClusterNodes.
- * @param values cluster nodes
- * @return list of hosts
- */
- public Iterable<String> getHostNamesList(Collection<ClusterNode> values) {
- List<String> hosts = new ArrayList<>();
- for (ClusterNode cn : values) {
- hosts.add(cn.hostname);
- }
- return hosts;
- }
-
- /**
- * Return a list of IPs based on current ClusterNodes.
- * @param values cluster nodes
- * @return list of hosts
- */
- public Iterable<String> getIPsList(Collection<ClusterNode> values) {
- List<String> hosts = new ArrayList<>();
- for (ClusterNode cn : values) {
- hosts.add(cn.ip);
- }
- return hosts;
- }
-
- /**
* Update ServiceRecord in Registry with IP and hostname.
* @param amState access to AM state
* @param yarnRegistry acces to YARN registry
@@ -1148,27 +1123,30 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
* Publish an export group.
* @param exportGroup export groups
* @param amState access to AM state
- * @param roleGroup component group
+ * @param groupName export group name
*/
- public void publishExportGroup(Map<String, List<ExportEntry>> exportGroup,
- StateAccessForProviders amState, String roleGroup) {
+ public void publishExportGroup(
+ Map<String, Set<ExportEntry>> exportGroup,
+ StateAccessForProviders amState, String groupName) {
// Publish in old format for the time being
Map<String, String> simpleEntries = new HashMap<>();
- for (Entry<String, List<ExportEntry>> entry : exportGroup.entrySet()) {
- List<ExportEntry> exports = entry.getValue();
+ for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) {
+ Set<ExportEntry> exports = entry.getValue();
if (SliderUtils.isNotEmpty(exports)) {
- // there is no support for multiple exports per name, so extract only
- // the first one
- simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
+ Set<String> values = new TreeSet<>();
+ for (ExportEntry export : exports) {
+ values.add(export.getValue());
+ }
+ simpleEntries.put(entry.getKey(), StringUtils.join(",", values));
}
}
- publishApplicationInstanceData(roleGroup, roleGroup,
+ publishApplicationInstanceData(groupName, groupName,
simpleEntries.entrySet(), amState);
- PublishedExports exports = new PublishedExports(roleGroup);
+ PublishedExports exports = new PublishedExports(groupName);
exports.setUpdated(new Date().getTime());
exports.putValues(exportGroup.entrySet());
- amState.getPublishedExportsSet().put(roleGroup, exports);
+ amState.getPublishedExportsSet().put(groupName, exports);
}
public Map<String, String> getExports(ConfTreeOperations appConf,
@@ -1179,75 +1157,26 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
return exports;
}
- private static final String COMPONENT_TAG = "component";
- private static final String HOST_FOLDER_FORMAT = "%s:%s";
- private static final String CONTAINER_LOGS_TAG = "container_log_dirs";
- private static final String CONTAINER_PWDS_TAG = "container_work_dirs";
-
- /**
- * Format the folder locations and publish in the registry service.
- * @param folders folder information
- * @param containerId container ID
- * @param hostFqdn host FQDN
- * @param componentName component name
- */
- public void publishFolderPaths(Map<String, String> folders,
- String containerId, String componentName, String hostFqdn,
- StateAccessForProviders amState,
- Map<String, ExportEntry> logFolderExports,
- Map<String, ExportEntry> workFolderExports) {
- Date now = new Date();
- for (Map.Entry<String, String> entry : folders.entrySet()) {
- ExportEntry exportEntry = new ExportEntry();
- exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn,
- entry.getValue()));
- exportEntry.setContainerId(containerId);
- exportEntry.setLevel(COMPONENT_TAG);
- exportEntry.setTag(componentName);
- exportEntry.setUpdatedTime(now.toString());
- if (entry.getKey().equals("AGENT_LOG_ROOT") ||
- entry.getKey().equals("LOG_DIR")) {
- synchronized (logFolderExports) {
- logFolderExports.put(containerId, exportEntry);
- }
- } else {
- synchronized (workFolderExports) {
- workFolderExports.put(containerId, exportEntry);
- }
- }
- log.info("Updating log and pwd folders for container {}", containerId);
- }
-
- PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
- exports.setUpdated(now.getTime());
- synchronized (logFolderExports) {
- updateExportsFromList(exports, logFolderExports);
- }
- amState.getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);
+ public String getGroupKey(String roleGroup, ConfTreeOperations appConf) {
+ String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
+ return getNameOrGroupKey(rolePrefix, roleGroup);
+ }
- exports = new PublishedExports(CONTAINER_PWDS_TAG);
- exports.setUpdated(now.getTime());
- synchronized (workFolderExports) {
- updateExportsFromList(exports, workFolderExports);
- }
- amState.getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
+ public String getNameKey(String roleName, String roleGroup,
+ ConfTreeOperations appConf) {
+ String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
+ return getNameOrGroupKey(rolePrefix, roleName);
}
- /**
- * Update the export data from the map.
- * @param exports published exports
- * @param folderExports folder exports
- */
- private void updateExportsFromList(PublishedExports exports,
- Map<String, ExportEntry> folderExports) {
- Map<String, List<ExportEntry>> perComponentList = new HashMap<>();
- for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) {
- String componentName = logEntry.getValue().getTag();
- if (!perComponentList.containsKey(componentName)) {
- perComponentList.put(componentName, new ArrayList<ExportEntry>());
+ public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) {
+ if (!rolePrefix.isEmpty()) {
+ if (!roleNameOrGroup.startsWith(rolePrefix)) {
+ log.warn("Something went wrong, {} doesn't start with {}",
+ roleNameOrGroup, rolePrefix);
+ return null;
}
- perComponentList.get(componentName).add(logEntry.getValue());
+ roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length());
}
- exports.putValues(perComponentList.entrySet());
+ return roleNameOrGroup.toUpperCase(Locale.ENGLISH);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 5edfa6a..63416cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -46,27 +46,24 @@ import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
-import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
public class DockerProviderService extends AbstractProviderService implements
ProviderCore,
DockerKeys,
@@ -77,10 +74,16 @@ public class DockerProviderService extends AbstractProviderService implements
private static final ProviderUtils providerUtils = new ProviderUtils(log);
private static final String EXPORT_GROUP = "quicklinks";
private static final String APPLICATION_TAG = "application";
+ private static final String HOST_KEY_FORMAT = "${%s_HOST}";
+ private static final String IP_KEY_FORMAT = "${%s_IP}";
+ private static final String VARIABLE_INDICATOR = "${";
private String clusterName = null;
private SliderFileSystem fileSystem = null;
+ private final Map<String, Set<ExportEntry>> exportMap =
+ new ConcurrentHashMap<>();
+
protected DockerProviderService() {
super("DockerProviderService");
}
@@ -118,9 +121,6 @@ public class DockerProviderService extends AbstractProviderService implements
String roleName = providerRole.name;
String roleGroup = providerRole.group;
- initializeApplicationConfiguration(instanceDefinition, fileSystem,
- roleGroup);
-
log.info("Build launch context for Docker");
log.debug(instanceDefinition.toString());
@@ -280,6 +280,23 @@ public class DockerProviderService extends AbstractProviderService implements
}
@Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ if (containerId != null) {
+ String containerIdStr = containerId.toString();
+ log.info("Removing container exports for {}", containerIdStr);
+ for (Set<ExportEntry> exportEntries : exportMap.values()) {
+ for (Iterator<ExportEntry> iter = exportEntries.iterator();
+ iter.hasNext();) {
+ ExportEntry entry = iter.next();
+ if (containerIdStr.equals(entry.getContainerId())) {
+ iter.remove();
+ }
+ }
+ }
+ }
+ }
+
+ @Override
public boolean processContainerStatus(ContainerId containerId,
ContainerStatus status) {
log.debug("Handling container status: {}", status);
@@ -301,7 +318,7 @@ public class DockerProviderService extends AbstractProviderService implements
containerIdStr, roleName, status.getIPs(), status.getHost());
publishExportGroups(containerIdStr, roleName, roleGroup,
- status.getHost());
+ status.getHost(), status.getIPs());
return false;
}
@@ -312,17 +329,13 @@ public class DockerProviderService extends AbstractProviderService implements
* are substituted with the actual hostnames of the containers.
*/
protected void publishExportGroups(String containerId,
- String roleName, String roleGroup, String thisHost) {
+ String roleName, String roleGroup, String thisHost, List<String> ips) {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
Map<String, String> exports = providerUtils.getExports(
getAmState().getAppConfSnapshot(), roleGroup);
- String hostKeyFormat = "${%s_HOST}";
- String hostNameKeyFormat = "${%s_HOSTNAME}";
- String ipKeyFormat = "${%s_IP}";
-
// publish export groups if any
Map<String, String> standardTokens = providerUtils.getStandardTokenMap(
appConf, internalsConf, roleName, roleGroup, containerId,
@@ -331,38 +344,33 @@ public class DockerProviderService extends AbstractProviderService implements
appConf.getComponent(roleGroup).options, standardTokens);
replaceTokens.putAll(standardTokens);
- String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
- for (Map.Entry<String, Map<String, ClusterNode>> entry :
- getAmState().getRoleClusterNodeMapping().entrySet()) {
- String otherRolePrefix = appConf.getComponentOpt(entry.getKey(),
- ROLE_PREFIX, "");
- if (!otherRolePrefix.equals(rolePrefix)) {
- // hostname replacements are only made within role prefix groups
- continue;
- }
- String key = entry.getKey();
- if (!rolePrefix.isEmpty()) {
- if (!key.startsWith(rolePrefix)) {
- log.warn("Something went wrong, {} doesn't start with {}", key,
- rolePrefix);
- continue;
- }
- key = key.substring(rolePrefix.length());
+ String roleNameKey = providerUtils.getNameKey(roleName, roleGroup,
+ appConf);
+ String roleNameIPKey = null;
+ if (roleNameKey != null) {
+ replaceTokens.put(String.format(HOST_KEY_FORMAT, roleNameKey), thisHost);
+ roleNameIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, roleNameKey));
+ } else {
+ // should not happen, but log if it does
+ log.info("Not replacing HOST or IP tokens because key was null for {}",
+ roleName);
+ }
+ String roleGroupKey = providerUtils.getGroupKey(roleGroup, appConf);
+ String roleGroupIPKey = null;
+ if (roleGroupKey != null) {
+ if (roleNameKey == null || !roleGroupKey.equals(roleNameKey)) {
+ replaceTokens.put(String.format(HOST_KEY_FORMAT, roleGroupKey),
+ thisHost);
+ roleGroupIPKey = Pattern.quote(String.format(IP_KEY_FORMAT,
+ roleGroupKey));
}
- key = key.toUpperCase(Locale.ENGLISH);
- String host = providerUtils.getHostsList(
- entry.getValue().values(), true).iterator().next();
- replaceTokens.put(String.format(hostKeyFormat, key), host);
- String hostName = providerUtils.getHostNamesList(
- entry.getValue().values()).iterator().next();
- replaceTokens.put(String.format(hostNameKeyFormat, key), hostName);
- String ip = providerUtils.getIPsList(
- entry.getValue().values()).iterator().next();
- replaceTokens.put(String.format(ipKeyFormat, key), ip);
+ } else {
+ // should not happen, but log if it does
+ log.info("Not replacing HOST or IP tokens because key was null for {}",
+ roleGroup);
}
replaceTokens.put("${THIS_HOST}", thisHost);
- Map<String, List<ExportEntry>> entries = new HashMap<>();
for (Entry<String, String> export : exports.entrySet()) {
String value = export.getValue();
// replace host names and site properties
@@ -372,18 +380,53 @@ public class DockerProviderService extends AbstractProviderService implements
value = value.replaceAll(Pattern.quote(token), entry.getValue());
}
}
- ExportEntry entry = new ExportEntry();
- entry.setLevel(APPLICATION_TAG);
- entry.setValue(value);
- entry.setUpdatedTime(new Date().toString());
- // over-write, app exports are singletons
- entries.put(export.getKey(), new ArrayList(Arrays.asList(entry)));
- log.info("Preparing to publish. Key {} and Value {}",
- export.getKey(), value);
+ Set<String> values = new HashSet<>();
+ for (String ip : ips) {
+ values.add(substituteIP(roleNameIPKey, roleGroupIPKey, ip, value));
+ }
+ for (String exportValue : values) {
+ if (exportValue.contains(VARIABLE_INDICATOR)) {
+ // not all variables have been substituted, so do not export
+ continue;
+ }
+ ExportEntry entry = new ExportEntry();
+ entry.setContainerId(containerId);
+ entry.setLevel(APPLICATION_TAG);
+ entry.setValue(exportValue);
+ entry.setUpdatedTime(new Date().toString());
+ Set<ExportEntry> exportEntries = getExportEntries(export.getKey());
+ exportEntries.add(entry);
+ log.info("Preparing to publish for {}. Key {} and Value {}",
+ roleName, export.getKey(), entry);
+ }
+ }
+ if (!exportMap.isEmpty()) {
+ providerUtils.publishExportGroup(exportMap, getAmState(), EXPORT_GROUP);
}
- if (!entries.isEmpty()) {
- providerUtils.publishExportGroup(entries, getAmState(), EXPORT_GROUP);
+ }
+
+ protected String substituteIP(String roleNameIPKey, String roleGroupIPKey,
+ String ip, String value) {
+ if (roleNameIPKey != null) {
+ value = value.replaceAll(roleNameIPKey, ip);
}
+ if (roleGroupIPKey != null) {
+ value = value.replaceAll(roleGroupIPKey, ip);
+ }
+ return value;
+ }
+
+ protected Set<ExportEntry> getExportEntries(String key) {
+ if (!this.exportMap.containsKey(key)) {
+ synchronized (this.exportMap) {
+ if (!this.exportMap.containsKey(key)) {
+ this.exportMap.put(key, Collections.newSetFromMap(
+ new ConcurrentHashMap<>()));
+ }
+ }
+ }
+
+ return this.exportMap.get(key);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcf0ff62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index c3b9b6f..2f99b27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedExports;
@@ -42,6 +41,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
@@ -291,12 +291,15 @@ public class IndexBlock extends SliderHamletBlock {
LI<UL<Hamlet>> item = ul.li();
item.span().$class("bold")._(export.description)._();
UL sublist = item.ul();
- for (Entry<String, List<ExportEntry>> entry : export.entries.entrySet()) {
- LI sublistItem = sublist.li()._(entry.getKey());
- for (ExportEntry exportEntry : entry.getValue()) {
- sublistItem._(exportEntry.getValue());
+ for (Entry<String, Set<ExportEntry>> entry : export.sortedEntries()
+ .entrySet()) {
+ if (SliderUtils.isNotEmpty(entry.getValue())) {
+ LI sublistItem = sublist.li()._(entry.getKey());
+ for (ExportEntry exportEntry : entry.getValue()) {
+ sublistItem._(exportEntry.getValue());
+ }
+ sublistItem._();
}
- sublistItem._();
}
sublist._();
item._();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org