You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/16 00:34:54 UTC
[2/2] git commit: SLIDER-481. Exports should allow a multiple line
items per export and a more hierarchical structure
SLIDER-481. Exports should allow a multiple line items per export and a more hierarchical structure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1ba58cd0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1ba58cd0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1ba58cd0
Branch: refs/heads/develop
Commit: 1ba58cd041221c47aacca77a5ff06246fa3f2a84
Parents: 08d7aec
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Oct 15 15:34:39 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Oct 15 15:34:39 2014 -0700
----------------------------------------------------------------------
app-packages/memcached/metainfo.xml | 18 +-
.../org/apache/slider/client/SliderClient.java | 102 +++++++
.../common/params/ActionRegistryArgs.java | 19 +-
.../apache/slider/common/params/Arguments.java | 2 +
.../core/registry/docstore/ExportEntry.java | 120 +++++++++
.../docstore/PublishedConfiguration.java | 2 +
.../registry/docstore/PublishedExports.java | 139 ++++++++++
.../docstore/PublishedExportsOutputter.java | 104 +++++++
.../registry/docstore/PublishedExportsSet.java | 100 +++++++
.../registry/info/CustomRegistryConstants.java | 3 +
.../registry/retrieve/RegistryRetriever.java | 91 ++++++-
.../providers/agent/AgentProviderService.java | 270 ++++++++++++++++---
.../agent/application/metadata/Component.java | 9 +
.../application/metadata/MetainfoParser.java | 1 +
.../slideram/SliderAMProviderService.java | 6 +
.../appmaster/state/ProviderAppState.java | 7 +
.../state/StateAccessForProviders.java | 7 +
.../server/appmaster/web/rest/RestPaths.java | 1 +
.../web/rest/publisher/PublisherResource.java | 31 ++-
.../slider/client/TestClientBadArgs.groovy | 43 +++
.../agent/TestAgentProviderService.java | 186 ++++++++++++-
.../framework/AgentCommandTestBase.groovy | 27 ++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 40 +++
23 files changed, 1267 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 5801ad2..0984dc9 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -23,17 +23,23 @@
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
<exportedConfigs>None</exportedConfigs>
+ <exportGroups>
+ <exportGroup>
+ <name>Servers</name>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${MEMCACHED_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
<components>
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
- <componentExports>
- <componentExport>
- <name>host_port</name>
- <value>${THIS_HOST}:${site.global.listen_port}</value>
- </componentExport>
- </componentExports>
+ <compExports>Servers-host_port</compExports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 99fde78..7c73394 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -117,6 +117,9 @@ import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
@@ -2331,11 +2334,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (registryArgs.listConf) {
// list the configurations
actionRegistryListConfigsYarn(registryArgs);
+ } else if (registryArgs.listExports) {
+ // list the exports
+ actionRegistryListExports(registryArgs);
} else if (SliderUtils.isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
+ } else if (SliderUtils.isSet(registryArgs.getExport)) {
+ // get a export group
+ PublishedExports publishedExports =
+ actionRegistryGetExport(registryArgs);
+ outputExport(publishedExports, registryArgs);
} else {
// it's an unknown command
log.info(ActionRegistryArgs.USAGE);
@@ -2744,6 +2755,34 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
+ * list exports available for an instance
+ *
+ * @param registryArgs registry Arguments
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ public void actionRegistryListExports(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+
+ RegistryRetriever retriever = new RegistryRetriever(instance);
+ PublishedExportsSet exports =
+ retriever.getExports(!registryArgs.internal);
+
+ for (String exportName : exports.keys()) {
+ if (!registryArgs.verbose) {
+ log.info("{}", exportName);
+ } else {
+ PublishedExports published =
+ exports.get(exportName);
+ log.info("{} : {}",
+ exportName,
+ published.description);
+ }
+ }
+ }
+
+ /**
* list configs available for an instance
*
* @param registryArgs registry Arguments
@@ -2768,6 +2807,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
+ * get a specific export group
+ *
+ * @param registryArgs registry Arguments
+ *
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ * @throws FileNotFoundException if the config is not found
+ */
+ @VisibleForTesting
+ public PublishedExports actionRegistryGetExport(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+
+ RegistryRetriever retriever = new RegistryRetriever(instance);
+ boolean external = !registryArgs.internal;
+ PublishedExportsSet exports =
+ retriever.getExports(external);
+
+ PublishedExports published = retriever.retrieveExports(exports,
+ registryArgs.getExport,
+ external);
+ return published;
+ }
+
+ /**
* write out the config. If a destination is provided and that dir is a
* directory, the entry is written to it with the name provided + extension,
* else it is printed to standard out.
@@ -2807,6 +2871,44 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
+ * write out the config
+ * @param published
+ * @param registryArgs
+ * @throws BadCommandArgumentsException
+ * @throws IOException
+ */
+ private void outputExport(PublishedExports published,
+ ActionRegistryArgs registryArgs) throws
+ BadCommandArgumentsException,
+ IOException {
+ // decide whether or not to print
+ String entry = registryArgs.getExport;
+ String format = ConfigFormat.JSON.toString();
+ ConfigFormat configFormat = ConfigFormat.resolve(format);
+ if (configFormat == null || configFormat != ConfigFormat.JSON) {
+ throw new BadCommandArgumentsException(
+ "Unknown/Unsupported format %s . Only JSON is supported.", format);
+ }
+
+ PublishedExportsOutputter outputter =
+ PublishedExportsOutputter.createOutputter(configFormat,
+ published);
+ boolean print = registryArgs.out == null;
+ if (!print) {
+ File destFile;
+ destFile = registryArgs.out;
+ if (destFile.isDirectory()) {
+ // creating it under a directory
+ destFile = new File(destFile, entry + "." + format);
+ }
+ log.info("Destination path: {}", destFile);
+ outputter.save(destFile);
+ } else {
+ print(outputter.asString());
+ }
+ }
+
+ /**
* Look up an instance
* @return instance data
* @throws SliderException other failures
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index deae4eb..1e4aba5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -47,8 +47,10 @@ public class ActionRegistryArgs extends AbstractActionArgs {
+ " ("
+ Arguments.ARG_LIST + "|"
+ Arguments.ARG_LISTCONF + "|"
+ + Arguments.ARG_LISTEXP + "|"
+ Arguments.ARG_LISTFILES + "|"
- + Arguments.ARG_GETCONF + "> "
+ + Arguments.ARG_GETCONF + "|"
+ + Arguments.ARG_GETEXP + "> "
+ Arguments.ARG_NAME + " <name> "
+ " )"
+ "[" + Arguments.ARG_VERBOSE + "] "
@@ -56,6 +58,8 @@ public class ActionRegistryArgs extends AbstractActionArgs {
+ "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ + System.getProperty("line.separator")
+ + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
;
public ActionRegistryArgs() {
}
@@ -90,7 +94,14 @@ public class ActionRegistryArgs extends AbstractActionArgs {
description = "get configuration")
public String getConf;
- @Parameter(names = {ARG_LISTFILES},
+ @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+ public boolean listExports;
+ @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+ public String getExport;
+
+ @Parameter(names = {ARG_LISTFILES},
description = "list files")
public String listFiles;
@@ -135,8 +146,8 @@ public class ActionRegistryArgs extends AbstractActionArgs {
super.validate();
//verify that at most one of the operations is set
- int gets = s(getConf) + s(getFiles);
- int lists = s(list) + s(listConf) + s(listFiles);
+ int gets = s(getConf) + s(getFiles) + s(getExport);
+ int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
int set = lists + gets;
if (set > 1) {
throw new UsageException(USAGE);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 24ad442..2f7af70 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -52,6 +52,7 @@ public interface Arguments {
String ARG_FORMAT = "--format";
String ARG_FORCE = "--force";
String ARG_GETCONF = "--getconf";
+ String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HELP = "--help";
String ARG_ID = "--id";
@@ -60,6 +61,7 @@ public interface Arguments {
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
+ String ARG_LISTEXP = "--listexp";
String ARG_LISTFILES = "--listfiles";
String ARG_LIVE = "--live";
String ARG_MANAGER = "--manager";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
new file mode 100644
index 0000000..4bcf6c1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ExportEntry {
+
+ /**
+ * The value of the export
+ */
+ private String value;
+ /**
+ * The container id of the container that is responsible for the export
+ */
+ private String containerId;
+ /**
+ * Tag associated with the container - its usually an identifier different than container id
+ * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ...
+ */
+ private String tag;
+ /**
+ * An export can be at the level of a component or an application
+ */
+ private String level;
+ /**
+ * The time when the export was updated
+ */
+ private String updatedTime;
+ /**
+ * The time when the export expires
+ */
+ private String validUntil;
+
+ public ExportEntry() {
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+ public String getUpdatedTime() {
+ return updatedTime;
+ }
+
+ public void setUpdatedTime(String updatedTime) {
+ this.updatedTime = updatedTime;
+ }
+
+ public String getValidUntil() {
+ return validUntil;
+ }
+
+ public void setValidUntil(String validUntil) {
+ this.validUntil = validUntil;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder("ExportEntry{").
+ append("value='").append(value).append("',").
+ append("containerId='").append(containerId).append("',").
+ append("tag='").append(tag).append("',").
+ append("level='").append(level).append("'").
+ append("updatedTime='").append(updatedTime).append("'").
+ append("validUntil='").append(validUntil).append("'").
+ append(" }").toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index cbc46f0..28f9d3d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.core.exceptions.BadConfigException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.io.IOException;
@@ -155,6 +156,7 @@ public class PublishedConfiguration {
*/
public String asJson() throws IOException {
ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
String json = mapper.writeValueAsString(entries);
return json;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
new file mode 100644
index 0000000..1919bfa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExports {
+
+ public String description;
+ public long updated;
+ public String updatedTime;
+ public Map<String, List<ExportEntry>> entries = new HashMap<String, List<ExportEntry>>();
+
+ public PublishedExports() {
+ }
+
+ /**
+ * build an empty published configuration
+ *
+ * @param description configuration description
+ */
+ public PublishedExports(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ *
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedExports(String description,
+ Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Is the configuration empty. This means either that it has not been given any values, or it is stripped down copy
+ * set down over the wire.
+ *
+ * @return
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration and Java properties object). Any existing
+ * value set is discarded
+ *
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.entries = new HashMap<String, List<ExportEntry>>();
+ for (Map.Entry<String, List<ExportEntry>> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Return the values as json string
+ *
+ * @return
+ *
+ * @throws IOException
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+ /**
+ * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's
+ * values
+ *
+ * @return the copy
+ */
+ public PublishedExports shallowCopy() {
+ PublishedExports that = new PublishedExports();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
new file mode 100644
index 0000000..b21e717
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Output a published configuration */
+public abstract class PublishedExportsOutputter {
+
+ protected final PublishedExports exports;
+
+ protected PublishedExportsOutputter(PublishedExports exports) {
+ this.exports = exports;
+ }
+
+ /**
+ * Create an outputter for the chosen format
+ *
+ * @param format format enumeration
+ * @param exports owning config
+ * @return the outputter
+ */
+
+ public static PublishedExportsOutputter createOutputter(ConfigFormat format,
+ PublishedExports exports) {
+ Preconditions.checkNotNull(exports);
+ switch (format) {
+ case JSON:
+ return new JsonOutputter(exports);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public void save(File dest) throws IOException {
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(dest);
+ save(out);
+ out.close();
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(out);
+ }
+ }
+
+ /**
+ * Save the content. The default saves the asString() value to the output stream
+ *
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+
+ /**
+ * Convert to a string
+ *
+ * @return
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ public static class JsonOutputter extends PublishedExportsOutputter {
+
+ public JsonOutputter(PublishedExports exports) {
+ super(exports);
+ }
+
+ @Override
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return exports.asJson();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
new file mode 100644
index 0000000..cdd35de
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExportsSet {
+
+ private static final PatternValidator validator = new PatternValidator(
+ RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+
+ public Map<String, PublishedExports> exports =
+ new HashMap<String, PublishedExports>();
+
+ public PublishedExportsSet() {
+ }
+
+ /**
+ * Put a name -it will be converted to lower case before insertion.
+ * Any existing entry will be overwritten (that includes an entry
+ * with a different case in the original name)
+ * @param name name of entry
+ * @param export published export
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void put(String name, PublishedExports export) {
+ String name1 = name.toLowerCase(Locale.ENGLISH);
+ validateName(name1);
+ exports.put(name1, export);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public static void validateName(String name) {
+ validator.validate(name);
+
+ }
+
+ public PublishedExports get(String name) {
+ return exports.get(name);
+ }
+
+ public boolean contains(String name) {
+ return exports.containsKey(name);
+ }
+
+ public int size() {
+ return exports.size();
+ }
+
+ public Set<String> keys() {
+ TreeSet<String> keys = new TreeSet<String>();
+ keys.addAll(exports.keySet());
+ return keys;
+ }
+
+ public PublishedExportsSet shallowCopy() {
+ PublishedExportsSet that = new PublishedExportsSet();
+ for (Map.Entry<String, PublishedExports> entry :
+ exports.entrySet()) {
+ that.put(entry.getKey(), entry.getValue().shallowCopy());
+ }
+ return that;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
index 65c122f..67b9feb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -35,6 +35,9 @@ public class CustomRegistryConstants {
public static final String PUBLISHER_CONFIGURATIONS_API =
"org.apache.slider.publisher.configurations";
+ public static final String PUBLISHER_EXPORTS_API =
+ "org.apache.slider.publisher.exports";
+
public static final String PUBLISHER_DOCUMENTS_API =
"org.apache.slider.publisher.documents";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 101efb2..a91f515 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -33,6 +33,8 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,6 +54,8 @@ public class RegistryRetriever {
private final String externalConfigurationURL;
private final String internalConfigurationURL;
+ private final String externalExportsURL;
+ private final String internalExportsURL;
private static final Client jerseyClient;
static {
@@ -63,9 +67,12 @@ public class RegistryRetriever {
jerseyClient.setFollowRedirects(true);
}
- public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL) {
- this.externalConfigurationURL = externalConfigurationURL;
- this.internalConfigurationURL = internalConfigurationURL;
+ public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL,
+ String externalExportsURL, String internalExportsURL) {
+ this.externalConfigurationURL = externalConfigurationURL;
+ this.internalConfigurationURL = internalConfigurationURL;
+ this.externalExportsURL = externalExportsURL;
+ this.internalExportsURL = internalExportsURL;
}
/**
@@ -99,6 +106,29 @@ public class RegistryRetriever {
}
}
externalConfigurationURL = url;
+
+ internal = record.getInternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API);
+ url = null;
+ if (internal != null) {
+ List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ internalExportsURL = url;
+ external = record.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API);
+ url = null;
+ if (external != null) {
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
+ }
+ externalExportsURL = url;
}
/**
@@ -138,6 +168,33 @@ public class RegistryRetriever {
return confURL;
}
+ protected String getExportURL(boolean external) throws FileNotFoundException {
+ String confURL = external ? externalExportsURL: internalExportsURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL");
+ }
+ return confURL;
+ }
+
+ /**
+ * Get the configurations of the registry
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return the configuration sets
+ */
+ public PublishedExportsSet getExports(boolean external) throws
+ FileNotFoundException, IOException {
+
+ String exportsUrl = getExportURL(external);
+ try {
+ WebResource webResource = jsonResource(exportsUrl);
+ log.debug("GET {}", exportsUrl);
+ PublishedExportsSet exportSet = webResource.get(PublishedExportsSet.class);
+ return exportSet;
+ } catch (UniformInterfaceException e) {
+ throw ExceptionConverter.convertJerseyException(exportsUrl, e);
+ }
+ }
+
private WebResource resource(String url) {
WebResource resource = jerseyClient.resource(url);
return resource;
@@ -174,7 +231,33 @@ public class RegistryRetriever {
throw ExceptionConverter.convertJerseyException(confURL, e);
}
}
-
+
+ /**
+ * Get a complete export, with all values
+ * @param exportSet
+ * @param name name of the configuration
+ * @param external flag to indicate that it is an external configuration
+ * @return the retrieved config
+ * @throws IOException IO problems
+ */
+ public PublishedExports retrieveExports(PublishedExportsSet exportSet,
+ String name,
+ boolean external) throws IOException {
+ if (!exportSet.contains(name)) {
+ throw new FileNotFoundException("Unknown export " + name);
+ }
+ String exportsURL = getExportURL(external);
+ exportsURL = SliderUtils.appendToURL(exportsURL, name);
+ try {
+ WebResource webResource = jsonResource(exportsURL);
+ PublishedExports publishedExports =
+ webResource.get(PublishedExports.class);
+ return publishedExports;
+ } catch (UniformInterfaceException e) {
+ throw ExceptionConverter.convertJerseyException(exportsURL, e);
+ }
+ }
+
@Override
public String toString() {
return super.toString()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 19b5ddd..fc7d935 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -52,7 +52,9 @@ import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCore;
@@ -95,10 +97,13 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -106,6 +111,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -127,10 +133,15 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
+ private static final String HOST_FOLDER_FORMAT = "%s:%s";
+ private static final String CONTAINER_LOGS_TAG = "container_log_dirs";
+ private static final String CONTAINER_PWDS_TAG = "container_work_dirs";
+ private static final String COMPONENT_TAG = "component";
+ private static final String APPLICATION_TAG = "application";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
private static final String SHARED_PORT_TAG = "SHARED";
private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
- private static final int MAX_LOG_ENTRIES = 20;
+ private static final int MAX_LOG_ENTRIES = 40;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -149,16 +160,25 @@ public class AgentProviderService extends AbstractProviderService implements
new ConcurrentHashMap<String, ComponentInstanceState>();
private final Map<String, Map<String, String>> componentInstanceData =
new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, Map<String, String>> exportGroups =
- new ConcurrentHashMap<String, Map<String, String>>();
+ private final Map<String, Map<String, List<ExportEntry>>> exportGroups =
+ new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>();
private final Map<String, Map<String, String>> allocatedPorts =
new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, String> workFolders =
- Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
+
+ private final Map<String, ExportEntry> logFolderExports =
+ Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > MAX_LOG_ENTRIES;
+ }
+ });
+ private final Map<String, ExportEntry> workFolderExports =
+ Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
}
});
+ private final Map<String, Set<String>> containerExportsMap =
+ new HashMap<String, Set<String>>();
/**
* Create an instance of AgentProviderService
@@ -491,7 +511,7 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, String> folders = registration.getLogFolders();
if (folders != null && !folders.isEmpty()) {
- publishLogFolderPaths(folders, containerId, roleName, hostFqdn);
+ publishFolderPaths(folders, containerId, roleName, hostFqdn);
}
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
@@ -563,7 +583,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Component operation. Status: {}", result);
if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
- publishLogFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
+ publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
}
}
@@ -639,6 +659,10 @@ public class AgentProviderService extends AbstractProviderService implements
this.getAllocatedPorts(containerId).put(portname, portNo);
if (instance != null) {
try {
+ // if the returned value is not a single port number then there are no
+ // meaningful way for Slider to use it during export
+ // No need to error out as it may not be the responsibility of the component
+ // to allocate port or the component may need an array of ports
instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
} catch (NumberFormatException e) {
log.warn("Failed to parse {}: {}", portNo, e);
@@ -648,6 +672,7 @@ public class AgentProviderService extends AbstractProviderService implements
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
+ processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName);
// and update registration entries
if (instance != null) {
@@ -697,7 +722,7 @@ public class AgentProviderService extends AbstractProviderService implements
throw new IOException(e);
}
}
-
+
@Override
public void notifyContainerCompleted(ContainerId containerId) {
if (containerId != null) {
@@ -719,6 +744,25 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
}
+
+ synchronized (this.containerExportsMap) {
+ Set<String> containerExportSets = containerExportsMap.get(containerIdStr);
+ if (containerExportSets != null) {
+ for (String containerExportStr : containerExportSets) {
+ String[] parts = containerExportStr.split(":");
+ Map<String, List<ExportEntry>> exportGroup = getCurrentExports(parts[0]);
+ List<ExportEntry> exports = exportGroup.get(parts[1]);
+ List<ExportEntry> exportToRemove = new ArrayList<ExportEntry>();
+ for (ExportEntry export : exports) {
+ if (containerIdStr.equals(export.getContainerId())) {
+ exportToRemove.add(export);
+ }
+ }
+ exports.removeAll(exportToRemove);
+ }
+ containerExportsMap.remove(containerIdStr);
+ }
+ }
}
}
@@ -754,6 +798,16 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
+ protected Map<String, ExportEntry> getLogFolderExports() {
+ return logFolderExports;
+ }
+
+ @VisibleForTesting
+ protected Map<String, ExportEntry> getWorkFolderExports() {
+ return workFolderExports;
+ }
+
+ @VisibleForTesting
protected Metainfo getMetainfo() {
return this.metainfo;
}
@@ -901,15 +955,59 @@ public class AgentProviderService extends AbstractProviderService implements
* @param hostFqdn
* @param roleName
*/
- protected void publishLogFolderPaths(
+ protected void publishFolderPaths(
Map<String, String> folders, String containerId, String roleName, String hostFqdn) {
- for (Map.Entry<String, String> entry: folders.entrySet()) {
- workFolders.put(String.format("%s->%s->%s->%s", roleName, hostFqdn, entry.getKey(), containerId),
- entry.getValue());
+ Date now = new Date();
+ for (Map.Entry<String, String> entry : folders.entrySet()) {
+ ExportEntry exportEntry = new ExportEntry();
+ exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn, entry.getValue()));
+ exportEntry.setContainerId(containerId);
+ exportEntry.setLevel(COMPONENT_TAG);
+ exportEntry.setTag(roleName);
+ exportEntry.setUpdatedTime(now.toString());
+ if (entry.getKey().equals("AGENT_LOG_ROOT")) {
+ synchronized (logFolderExports) {
+ getLogFolderExports().put(containerId, exportEntry);
+ }
+ } else {
+ synchronized (workFolderExports) {
+ getWorkFolderExports().put(containerId, exportEntry);
+ }
+ }
+ log.info("Updating log and pwd folders for container {}", containerId);
}
- publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
- (new HashMap<String, String>(this.workFolders)).entrySet());
+ PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
+ exports.setUpdated(now.getTime());
+ synchronized (logFolderExports) {
+ updateExportsFromList(exports, getLogFolderExports());
+ }
+ getAmState().getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);
+
+ exports = new PublishedExports(CONTAINER_PWDS_TAG);
+ exports.setUpdated(now.getTime());
+ synchronized (workFolderExports) {
+ updateExportsFromList(exports, getWorkFolderExports());
+ }
+ getAmState().getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
+ }
+
+ /**
+ * Update the export data from the map
+ * @param exports
+ * @param folderExports
+ */
+ private void updateExportsFromList(PublishedExports exports, Map<String, ExportEntry> folderExports) {
+ Map<String, List<ExportEntry>> perComponentList = new HashMap<String, List<ExportEntry>>();
+ for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet())
+ {
+ String componentName = logEntry.getValue().getTag();
+ if(!perComponentList.containsKey(componentName)) {
+ perComponentList.put(componentName, new ArrayList<ExportEntry>());
+ }
+ perComponentList.get(componentName).add(logEntry.getValue());
+ }
+ exports.putValues(perComponentList.entrySet());
}
@@ -954,13 +1052,12 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- List<ExportGroup> exportGroups = application.getExportGroups();
- boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+ List<ExportGroup> appExportGroups = application.getExportGroups();
+ boolean hasExportGroups = appExportGroups != null && !appExportGroups.isEmpty();
Set<String> appExports = new HashSet();
String appExportsStr = getApplicationComponent(roleName).getAppExports();
- boolean hasNoAppExports = appExportsStr == null || appExportsStr.isEmpty();
- if (!hasNoAppExports) {
+ if (SliderUtils.isSet(appExportsStr)) {
for (String appExport : appExportsStr.split(",")) {
if (appExport.trim().length() > 0) {
appExports.add(appExport.trim());
@@ -988,11 +1085,12 @@ public class AgentProviderService extends AbstractProviderService implements
}
Set<String> modifiedGroups = new HashSet<String>();
- for (ExportGroup exportGroup : exportGroups) {
+ for (ExportGroup exportGroup : appExportGroups) {
List<Export> exports = exportGroup.getExports();
if (exports != null && !exports.isEmpty()) {
String exportGroupName = exportGroup.getName();
- Map<String, String> map = getCurrentExports(exportGroupName);
+ ConcurrentHashMap<String, List<ExportEntry>> map =
+ (ConcurrentHashMap<String, List<ExportEntry>>)getCurrentExports(exportGroupName);
for (Export export : exports) {
if (canBeExported(exportGroupName, export.getName(), appExports)) {
String value = export.getValue();
@@ -1002,7 +1100,12 @@ public class AgentProviderService extends AbstractProviderService implements
value = value.replace(token, replaceTokens.get(token));
}
}
- map.put(export.getName(), value);
+ ExportEntry entry = new ExportEntry();
+ entry.setLevel(APPLICATION_TAG);
+ entry.setValue(value);
+ entry.setUpdatedTime(new Date().toString());
+ // over-write, app exports are singletons
+ map.put(export.getName(), new ArrayList(Arrays.asList(entry)));
log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
}
@@ -1024,11 +1127,11 @@ public class AgentProviderService extends AbstractProviderService implements
return appExports.contains(String.format("%s-%s", exportGroupName, name));
}
- protected Map<String, String> getCurrentExports(String groupName) {
+ protected Map<String, List<ExportEntry>> getCurrentExports(String groupName) {
if (!this.exportGroups.containsKey(groupName)) {
synchronized (this.exportGroups) {
if (!this.exportGroups.containsKey(groupName)) {
- this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
+ this.exportGroups.put(groupName, new ConcurrentHashMap<String, List<ExportEntry>>());
}
}
}
@@ -1037,10 +1140,24 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void publishModifiedExportGroups(Set<String> modifiedGroups) {
- synchronized (this.exportGroups) {
- for (String groupName : modifiedGroups) {
- publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet());
+ for (String groupName : modifiedGroups) {
+ Map<String, List<ExportEntry>> entries = this.exportGroups.get(groupName);
+
+ // Publish in old format for the time being
+ Map<String, String> simpleEntries = new HashMap<String, String>();
+ for (Map.Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
+ List<ExportEntry> exports = entry.getValue();
+ if(exports != null && exports.size() > 0) {
+ // there is no support for multiple exports per name - so extract only the first one
+ simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
+ }
}
+ publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet());
+
+ PublishedExports exports = new PublishedExports(groupName);
+ exports.setUpdated(new Date().getTime());
+ exports.putValues(entries.entrySet());
+ getAmState().getPublishedExportsSet().put(groupName, exports);
}
}
@@ -1095,14 +1212,102 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ /** Publish component instance specific data if the component demands it */
+ protected void processAndPublishComponentSpecificExports(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String roleName) {
+ String portVarFormat = "${site.%s}";
+ String hostNamePattern = "${" + roleName + "_HOST}";
+
+ List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups();
+ Component component = getMetainfo().getApplicationComponent(roleName);
+ if (component != null && SliderUtils.isSet(component.getCompExports())
+ && appExportGroups != null && appExportGroups.size() > 0) {
+
+ Set<String> compExports = new HashSet();
+ String compExportsStr = component.getCompExports();
+ for (String appExport : compExportsStr.split(",")) {
+ if (appExport.trim().length() > 0) {
+ compExports.add(appExport.trim());
+ }
+ }
+
+ Date now = new Date();
+ Set<String> modifiedGroups = new HashSet<String>();
+ for (ExportGroup exportGroup : appExportGroups) {
+ List<Export> exports = exportGroup.getExports();
+ if (exports != null && !exports.isEmpty()) {
+ String exportGroupName = exportGroup.getName();
+ ConcurrentHashMap<String, List<ExportEntry>> map =
+ (ConcurrentHashMap<String, List<ExportEntry>>) getCurrentExports(exportGroupName);
+ for (Export export : exports) {
+ if (canBeExported(exportGroupName, export.getName(), compExports)) {
+ log.info("Attempting to publish {} of group {} for component type {}",
+ export.getName(), exportGroupName, roleName);
+ String templateToExport = export.getValue();
+ for (String portName : ports.keySet()) {
+ boolean publishData = false;
+ String portValPattern = String.format(portVarFormat, portName);
+ if (templateToExport.contains(portValPattern)) {
+ templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
+ publishData = true;
+ }
+ if (templateToExport.contains(hostNamePattern)) {
+ templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
+ publishData = true;
+ }
+ if (publishData) {
+ ExportEntry entryToAdd = new ExportEntry();
+ entryToAdd.setLevel(COMPONENT_TAG);
+ entryToAdd.setValue(templateToExport);
+ entryToAdd.setUpdatedTime(now.toString());
+ entryToAdd.setContainerId(containerId);
+
+ List<ExportEntry> existingList =
+ map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd)));
+
+ // in-place edit, no lock needed
+ if (existingList != null) {
+ boolean updatedInPlace = false;
+ for (ExportEntry entry : existingList) {
+ if (containerId.equalsIgnoreCase(entry.getContainerId())) {
+ entryToAdd.setValue(templateToExport);
+ entryToAdd.setUpdatedTime(now.toString());
+ updatedInPlace = true;
+ }
+ }
+ if (!updatedInPlace) {
+ existingList.add(entryToAdd);
+ }
+ }
+
+ log.info("Publishing {} for name {} and container {}",
+ templateToExport, export.getName(), containerId);
+ modifiedGroups.add(exportGroupName);
+ synchronized (containerExportsMap) {
+ if (!containerExportsMap.containsKey(containerId)) {
+ containerExportsMap.put(containerId, new HashSet<String>());
+ }
+ Set<String> containerExportMaps = containerExportsMap.get(containerId);
+ containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName()));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ publishModifiedExportGroups(modifiedGroups);
+ }
+ }
+
private void publishComponentInstanceData() {
Map<String, String> dataToPublish = new HashMap<String, String>();
- synchronized (this.componentInstanceData) {
- for (String container : getComponentInstanceData().keySet()) {
- for (String prop : getComponentInstanceData().get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, getComponentInstanceData().get(container).get(prop));
- }
+ for (String container : getComponentInstanceData().keySet()) {
+ for (String prop : getComponentInstanceData().get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, getComponentInstanceData().get(container).get(prop));
}
}
publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
@@ -1616,5 +1821,4 @@ public class AgentProviderService extends AbstractProviderService implements
"");
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 9f3dd0f..418868c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -34,6 +34,7 @@ public class Component {
String maxInstanceCount;
String autoStartOnFailure;
String appExports;
+ String compExports;
CommandScript commandScript;
List<ComponentExport> componentExports;
@@ -82,6 +83,14 @@ public class Component {
this.appExports = appExports;
}
+ public String getCompExports() {
+ return compExports;
+ }
+
+ public void setCompExports(String compExports) {
+ this.compExports = compExports;
+ }
+
public String getMinInstanceCount() {
return minInstanceCount;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index c92c265..1d8403f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -61,6 +61,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/maxInstanceCount");
digester.addBeanPropertySetter("*/component/autoStartOnFailure");
digester.addBeanPropertySetter("*/component/appExports");
+ digester.addBeanPropertySetter("*/component/compExports");
digester.addObjectCreate("*/componentExport", ComponentExport.class);
digester.addBeanPropertySetter("*/componentExport/name");
digester.addBeanPropertySetter("*/componentExport/value");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index 601c3f9..afe6428 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -146,6 +146,8 @@ public class SliderAMProviderService extends AbstractProviderService implements
String configurationsURL = SliderUtils.appendToURL(
publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
+ String exportsURL = SliderUtils.appendToURL(
+ publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS);
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.webEndpoint(
@@ -166,6 +168,10 @@ public class SliderAMProviderService extends AbstractProviderService implements
RegistryTypeUtils.restEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
new URI(configurationsURL)));
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_EXPORTS_API,
+ new URI(exportsURL)));
} catch (URISyntaxException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index a0871ae..9c5da12 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.utility.PatternValidator;
@@ -40,6 +41,7 @@ public class ProviderAppState implements StateAccessForProviders {
private final Map<String, PublishedConfigSet> publishedConfigSets =
new ConcurrentHashMap<String, PublishedConfigSet>(5);
+ private final PublishedExportsSet publishedExportsSets = new PublishedExportsSet();
private static final PatternValidator validator = new PatternValidator(
RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
private String applicationName;
@@ -66,6 +68,11 @@ public class ProviderAppState implements StateAccessForProviders {
}
@Override
+ public PublishedExportsSet getPublishedExportsSet() {
+ return publishedExportsSets;
+ }
+
+ @Override
public PublishedConfigSet getPublishedConfigSet(String name) {
return publishedConfigSets.get(name);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 1714f75..b907b06 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -26,6 +26,7 @@ import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import java.util.Collection;
import java.util.List;
@@ -51,6 +52,12 @@ public interface StateAccessForProviders {
PublishedConfigSet getPublishedSliderConfigurations();
/**
+ * Get the published exports set
+ * @return
+ */
+ PublishedExportsSet getPublishedExportsSet();
+
+ /**
* Get a named published config set
* @param name name to look up
* @return the instance or null
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 93601ad..94f1e4c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -61,6 +61,7 @@ public class RestPaths {
= "[a-z0-9][a-z0-9_.\\+-]*";
public static final String SLIDER_CONFIGSET = "slider";
+ public static final String SLIDER_EXPORTS = "exports";
public static final String SLIDER_CLASSPATH = "classpath";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index 5d8b657..e47bbb9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -23,6 +23,8 @@ import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
import org.apache.slider.core.registry.docstore.UriMap;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -56,7 +58,10 @@ public class PublisherResource {
protected static final Logger log =
LoggerFactory.getLogger(PublisherResource.class);
private final WebAppApi slider;
- public static final String SET_NAME =
+ public static final String EXPORTS_NAME = "exports";
+ public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
+ public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + "/{exportname}" ;
+ public static final String SET_NAME =
"{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
private static final String CONFIG =
SET_NAME + "/{config: " + PUBLISHED_CONFIGURATION_REGEXP + "}";
@@ -101,7 +106,9 @@ public class PublisherResource {
UriMap uriMap = new UriMap();
for (String name : appState.listConfigSets()) {
uriMap.put(name, baseURL + name);
+ log.info("Tick tack {} and {}", name, baseURL);
}
+ uriMap.put(EXPORTS_NAME, baseURL + EXPORTS_NAME);
return uriMap;
}
@@ -114,6 +121,26 @@ public class PublisherResource {
}
@GET
+ @Path(EXPORTS_RESOURCES_PATH)
+ @Produces({MediaType.APPLICATION_JSON})
+ public PublishedExportsSet gePublishedExports() {
+
+ PublishedExportsSet set = appState.getPublishedExportsSet();
+ return set.shallowCopy();
+ }
+
+ @GET
+ @Path(EXPORT_RESOURCE_PATH)
+ @Produces({MediaType.APPLICATION_JSON})
+ public PublishedExports getAMExports2(@PathParam("exportname") String exportname,
+ @Context UriInfo uriInfo,
+ @Context HttpServletResponse res) {
+ init(res, uriInfo);
+ PublishedExportsSet set = appState.getPublishedExportsSet();
+ return set.get(exportname);
+ }
+
+ @GET
@Path("/"+ SET_NAME)
@Produces({MediaType.APPLICATION_JSON})
public PublishedConfigSet getPublishedConfiguration(
@@ -129,7 +156,7 @@ public class PublisherResource {
}
private void logRequest(UriInfo uriInfo) {
- log.debug(uriInfo.getRequestUri().toString());
+ log.info(uriInfo.getRequestUri().toString());
}
@GET
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ba58cd0/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
index d1f8a8f..7d596d6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
@@ -24,8 +24,10 @@ import org.apache.hadoop.conf.Configuration
import org.apache.slider.common.params.ActionRegistryArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.exceptions.UsageException
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.junit.Test
@@ -88,4 +90,45 @@ class TestClientBadArgs extends ServiceLauncherBaseTest {
log.info(exception.toString())
}
+ @Test
+ public void testRegistryExportBadUsage1() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Expected a value after parameter --getexp",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_GETEXP])
+ assert exception instanceof BadCommandArgumentsException
+ log.info(exception.toString())
+ }
+
+ @Test
+ public void testRegistryExportBadUsage2() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Expected a value after parameter --getexp",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP])
+ assert exception instanceof BadCommandArgumentsException
+ log.info(exception.toString())
+ }
+
+ @Test
+ public void testRegistryExportBadUsage3() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Usage: registry",
+ [SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP,
+ "export1"])
+ assert exception instanceof UsageException
+ log.info(exception.toString())
+ }
}