You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/07 21:10:22 UTC
[22/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported
from slider-core module. (jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
new file mode 100644
index 0000000..1bdfb9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Client code for interacting with a list of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YarnAppListClient {
+
+ private final SliderYarnClientImpl yarnClient;
+ private final String username;
+ private final Configuration conf;
+ private static final Logger log =
+ LoggerFactory.getLogger(YarnAppListClient.class);
+
+ public YarnAppListClient(SliderYarnClientImpl yarnClient,
+ String username,
+ Configuration conf) {
+
+ Preconditions.checkArgument(yarnClient != null,
+ "yarn client is null: is app inited?");
+ Preconditions.checkArgument(username != null,
+ "username is null");
+ Preconditions.checkArgument(conf != null,
+ "conf parameter is null");
+ this.yarnClient = yarnClient;
+ this.username = username;
+ this.conf = conf;
+ }
+
+ /**
+ * find all live instances of a specific app -if there is more than one
+ * in the cluster, this returns them all. State should be running or earlier
+ * in the lifecycle
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ public List<ApplicationReport> findAllLiveInstances(String appname)
+ throws YarnException, IOException {
+ return yarnClient.findAllLiveInstances(username, appname);
+ }
+
+
+ /**
+ * Find an instance of a application belong to the current user
+ * @param appname application name
+ * @return the app report or null if none is found
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ public ApplicationReport findInstance(String appname) throws
+ YarnException,
+ IOException {
+ List<ApplicationReport> instances = listInstances(null);
+ return yarnClient.findClusterInInstanceList(instances, appname);
+ }
+
+ /**
+ * List instances belonging to the specific user
+ * @return a possibly empty list of AMs
+ */
+ public List<ApplicationReport> listInstances()
+ throws YarnException, IOException {
+ return listInstances(null);
+ }
+
+ /**
+ * List instances belonging to a specific user
+ * @return a possibly empty list of AMs
+ * @param user user if not the default. null means default, "" means all users,
+ * otherwise it is the name of a user
+ */
+ public List<ApplicationReport> listInstances(String user)
+ throws YarnException, IOException {
+ String listUser = user == null ? username : user;
+ return yarnClient.listDeployedInstances(listUser);
+ }
+
+ /**
+ * Enumerate slider instances for the current user, and the
+ * most recent app report, where available.
+ * @param listOnlyInState boolean to indicate that the instances should
+ * only include those in a YARN state
+ * <code> minAppState <= currentState <= maxAppState </code>
+ *
+ * @param minAppState minimum application state to include in enumeration.
+ * @param maxAppState maximum application state to include
+ * @return a map of application instance name to description
+ * @throws IOException Any IO problem
+ * @throws YarnException YARN problems
+ */
+ public Map<String, SliderInstanceDescription> enumSliderInstances(
+ boolean listOnlyInState,
+ YarnApplicationState minAppState,
+ YarnApplicationState maxAppState)
+ throws IOException, YarnException {
+
+ CoreFileSystem sliderFileSystem = new CoreFileSystem(conf);
+ Preconditions.checkArgument(!listOnlyInState || minAppState != null,
+ "null minAppState when listOnlyInState set");
+ Preconditions.checkArgument(!listOnlyInState || maxAppState != null,
+ "null maxAppState when listOnlyInState set");
+ if (!listOnlyInState) {
+ // if there's not filtering, ask for the entire range of states
+ minAppState = YarnApplicationState.NEW;
+ maxAppState = YarnApplicationState.KILLED;
+ }
+ // get the complete list of persistent instances
+ Map<String, Path> persistentInstances =
+ sliderFileSystem.listPersistentInstances();
+ Map<String, SliderInstanceDescription> descriptions =
+ new HashMap<String, SliderInstanceDescription>(persistentInstances.size());
+
+ if (persistentInstances.isEmpty()) {
+ // an empty listing is a success if no cluster was named
+ log.debug("No application instances found");
+ return descriptions;
+ }
+
+ // enum those the RM knows about
+ List<ApplicationReport> rmInstances = listInstances();
+ SliderUtils.sortApplicationsByMostRecent(rmInstances);
+ Map<String, ApplicationReport> reportMap =
+ SliderUtils.buildApplicationReportMap(rmInstances, minAppState,
+ maxAppState);
+ log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
+ persistentInstances.size(),
+ rmInstances.size(),
+ minAppState, maxAppState,
+ reportMap.size());
+
+ // at this point there is a list of all persistent instances, and
+ // a (possibly filtered) list of application reports
+
+ for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) {
+ // loop through the persistent values
+ String name = entry.getKey();
+
+ // look up any report from the (possibly filtered) report set
+ ApplicationReport report = reportMap.get(name);
+ if (!listOnlyInState || report != null) {
+ // if the enum wants to filter in state, only add it if there is
+ // a report in that range. Otherwise: include all values
+ SliderInstanceDescription sid = new SliderInstanceDescription(
+ name, entry.getValue(), report);
+ descriptions.put(name, sid);
+ }
+ }
+
+ return descriptions;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
new file mode 100644
index 0000000..ddab606
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public enum ConfigFormat {
+
+ JSON("json"),
+ PROPERTIES("properties"),
+ XML("xml"),
+ HADOOP_XML("hadoop-xml"),
+ ENV("env"),
+ TEMPLATE("template"),
+ YAML("yaml"),
+ ;
+ ConfigFormat(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private final String suffix;
+
+ public String getSuffix() {
+ return suffix;
+ }
+
+
+ @Override
+ public String toString() {
+ return suffix;
+ }
+
+ /**
+ * Get a matching format or null
+ * @param type
+ * @return the format
+ */
+ public static ConfigFormat resolve(String type) {
+ for (ConfigFormat format: values()) {
+ if (format.getSuffix().equals(type)) {
+ return format;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
new file mode 100644
index 0000000..2e1615b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
+ public static String replaceProps(Map<String, String> config, String content) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+ }
+ String value = content;
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ return value;
+ }
+
+ public static Map<String, String> replacePropsInConfig(
+ Map<String, String> config, Map<String, String> env) {
+ Map<String, String> tokens = new HashMap<>();
+ for (Entry<String, String> entry : env.entrySet()) {
+ tokens.put("${" + entry.getKey() + "}", entry.getValue());
+ }
+ Map<String, String> newConfig = new HashMap<>();
+ for (Entry<String, String> entry : config.entrySet()) {
+ String value = entry.getValue();
+ for (Map.Entry<String,String> token : tokens.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ Matcher.quoteReplacement(token.getValue()));
+ }
+ newConfig.put(entry.getKey(), entry.getValue());
+ }
+ return newConfig;
+ }
+
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
new file mode 100644
index 0000000..88bac77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public class ConfigurationResolver {
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
new file mode 100644
index 0000000..4bcf6c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ExportEntry {
+
+ /**
+ * The value of the export
+ */
+ private String value;
+ /**
+ * The container id of the container that is responsible for the export
+ */
+ private String containerId;
+ /**
+ * Tag associated with the container - its usually an identifier different than container id
+ * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ...
+ */
+ private String tag;
+ /**
+ * An export can be at the level of a component or an application
+ */
+ private String level;
+ /**
+ * The time when the export was updated
+ */
+ private String updatedTime;
+ /**
+ * The time when the export expires
+ */
+ private String validUntil;
+
+ public ExportEntry() {
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ public String getLevel() {
+ return level;
+ }
+
+ public void setLevel(String level) {
+ this.level = level;
+ }
+ public String getUpdatedTime() {
+ return updatedTime;
+ }
+
+ public void setUpdatedTime(String updatedTime) {
+ this.updatedTime = updatedTime;
+ }
+
+ public String getValidUntil() {
+ return validUntil;
+ }
+
+ public void setValidUntil(String validUntil) {
+ this.validUntil = validUntil;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder("ExportEntry{").
+ append("value='").append(value).append("',").
+ append("containerId='").append(containerId).append("',").
+ append("tag='").append(tag).append("',").
+ append("level='").append(level).append("'").
+ append("updatedTime='").append(updatedTime).append("'").
+ append("validUntil='").append(validUntil).append("'").
+ append(" }").toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
new file mode 100644
index 0000000..edc129e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfigSet {
+
+ private static final PatternValidator validator = new PatternValidator(
+ RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+
+ public Map<String, PublishedConfiguration> configurations =
+ new HashMap<>();
+
+ public PublishedConfigSet() {
+ }
+
+ /**
+ * Put a name -it will be converted to lower case before insertion.
+ * Any existing entry will be overwritten (that includes an entry
+ * with a different case in the original name)
+ * @param name name of entry
+ * @param conf configuration
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void put(String name, PublishedConfiguration conf) {
+ String name1 = name.toLowerCase(Locale.ENGLISH);
+ validateName(name1);
+ configurations.put(name1, conf);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public static void validateName(String name) {
+ validator.validate(name);
+
+ }
+
+ public PublishedConfiguration get(String name) {
+ return configurations.get(name);
+ }
+
+ public boolean contains(String name) {
+ return configurations.containsKey(name);
+ }
+
+ public int size() {
+ return configurations.size();
+ }
+
+ public Set<String> keys() {
+ TreeSet<String> keys = new TreeSet<>();
+ keys.addAll(configurations.keySet());
+ return keys;
+ }
+
+ public PublishedConfigSet shallowCopy() {
+ PublishedConfigSet that = new PublishedConfigSet();
+ for (Map.Entry<String, PublishedConfiguration> entry :
+ configurations.entrySet()) {
+ that.put(entry.getKey(), entry.getValue().shallowCopy());
+ }
+ return that;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
new file mode 100644
index 0000000..50b522f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+ public String description;
+ public long updated;
+
+ public String updatedTime;
+
+ public Map<String, String> entries = new HashMap<>();
+
+ public PublishedConfiguration() {
+ }
+
+ /**
+ * build an empty published configuration
+ * @param description configuration description
+ */
+ public PublishedConfiguration(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Build a published configuration, using the keys from keysource,
+ * but resolving the values from the value source, via Configuration.get()
+ * @param description configuration description
+ * @param keysource source of keys
+ * @param valuesource source of values
+ */
+ public PublishedConfiguration(String description,
+ Iterable<Map.Entry<String, String>> keysource,
+ Configuration valuesource) {
+ this.description = description;
+ putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+ }
+
+
+ /**
+ * Is the configuration empty. This means either that it has not
+ * been given any values, or it is stripped down copy set down over the
+ * wire.
+ * @return true if it is empty
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration
+ * and Java properties object).
+ * Any existing value set is discarded
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, String>> entries) {
+ this.entries = new HashMap<String, String>();
+ for (Map.Entry<String, String> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+
+ }
+
+ /**
+ * Convert to Hadoop XML
+ * @return the configuration as a Hadoop Configuratin
+ */
+ public Configuration asConfiguration() {
+ Configuration conf = new Configuration(false);
+ try {
+ ConfigHelper.addConfigMap(conf, entries, "");
+ } catch (BadConfigException e) {
+ // triggered on a null value; switch to a runtime (and discard the stack)
+ throw new RuntimeException(e.toString());
+ }
+ return conf;
+ }
+
+ public String asConfigurationXML() throws IOException {
+ return ConfigHelper.toXml(asConfiguration());
+ }
+
+ /**
+ * Convert values to properties
+ * @return a property file
+ */
+ public Properties asProperties() {
+ Properties props = new Properties();
+ props.putAll(entries);
+ return props;
+ }
+
+ /**
+ * Return the values as json string
+ * @return the JSON representation
+ * @throws IOException marshalling failure
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+
+ /**
+ * This makes a copy without the nested content -so is suitable
+ * for returning as part of the list of a parent's values
+ * @return the copy
+ */
+ public PublishedConfiguration shallowCopy() {
+ PublishedConfiguration that = new PublishedConfiguration();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Create an outputter for a given format
+ * @param format format to use
+ * @return an instance of output
+ */
+ public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
+ return PublishedConfigurationOutputter.createOutputter(format, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
new file mode 100644
index 0000000..9bdcfcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Properties;
+
+/**
+ * Output a published configuration
+ */
+public abstract class PublishedConfigurationOutputter {
+
+ protected final PublishedConfiguration owner;
+
+ protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Save the config to a destination file, in the format of this outputter
+ * @param dest destination file
+ * @throws IOException
+ */
+/* JDK7
+ public void save(File dest) throws IOException {
+ try(FileOutputStream out = new FileOutputStream(dest)) {
+ save(out);
+ out.close();
+ }
+ }
+*/
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ /**
+ * Save the content. The default saves the asString() value
+ * to the output stream
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+ /**
+ * Convert to a string
+ * @return the string form
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ /**
+ * Create an outputter for the chosen format
+ * @param format format enumeration
+ * @param owner owning config
+ * @return the outputter
+ */
+
+ public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
+ PublishedConfiguration owner) {
+ Preconditions.checkNotNull(owner);
+ switch (format) {
+ case XML:
+ case HADOOP_XML:
+ return new XmlOutputter(owner);
+ case PROPERTIES:
+ return new PropertiesOutputter(owner);
+ case JSON:
+ return new JsonOutputter(owner);
+ case ENV:
+ return new EnvOutputter(owner);
+ case TEMPLATE:
+ return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public static class XmlOutputter extends PublishedConfigurationOutputter {
+
+
+ private final Configuration configuration;
+
+ public XmlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ configuration = owner.asConfiguration();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ configuration.writeXml(out);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return ConfigHelper.toXml(configuration);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+ }
+
+ public static class PropertiesOutputter extends PublishedConfigurationOutputter {
+
+ private final Properties properties;
+
+ public PropertiesOutputter(PublishedConfiguration owner) {
+ super(owner);
+ properties = owner.asProperties();
+ }
+
+ @Override
+ public void save(OutputStream out) throws IOException {
+ properties.store(out, "");
+ }
+
+
+ public String asString() throws IOException {
+ StringWriter sw = new StringWriter();
+ properties.store(sw, "");
+ return sw.toString();
+ }
+ }
+
+
+ public static class JsonOutputter extends PublishedConfigurationOutputter {
+
+ public JsonOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return owner.asJson();
+ }
+ }
+
+
+ public static class EnvOutputter extends PublishedConfigurationOutputter {
+
+ public EnvOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ if (!owner.entries.containsKey("content")) {
+ throw new IOException("Configuration has no content field and cannot " +
+ "be retrieved as type 'env'");
+ }
+ String content = owner.entries.get("content");
+ return ConfigUtils.replaceProps(owner.entries, content);
+ }
+ }
+
+ public static class TemplateOutputter extends EnvOutputter {
+ public TemplateOutputter(PublishedConfiguration owner) {
+ super(owner);
+ }
+ }
+
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
new file mode 100644
index 0000000..0759b4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExports {
+
+ public String description;
+ public long updated;
+ public String updatedTime;
+ public Map<String, List<ExportEntry>> entries = new HashMap<>();
+
+ public PublishedExports() {
+ }
+
+ /**
+ * build an empty published configuration
+ *
+ * @param description configuration description
+ */
+ public PublishedExports(String description) {
+ this.description = description;
+ }
+
+ /**
+ * Build a configuration from the entries
+ *
+ * @param description configuration description
+ * @param entries entries to put
+ */
+ public PublishedExports(String description,
+ Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.description = description;
+ putValues(entries);
+ }
+
+ /**
+ * Is the configuration empty. This means either that it has not been given any values,
+ * or it is stripped down copy
+ * set down over the wire.
+ *
+ * @return true if it is empty
+ */
+ public boolean isEmpty() {
+ return entries.isEmpty();
+ }
+
+ public long getUpdated() {
+ return updated;
+ }
+
+ public void setUpdated(long updated) {
+ this.updated = updated;
+ this.updatedTime = new Date(updated).toString();
+ }
+
+ /**
+ * Set the values from an iterable (this includes a Hadoop Configuration and Java properties
+ * object). Any existing value set is discarded
+ *
+ * @param entries entries to put
+ */
+ public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+ this.entries = new HashMap<String, List<ExportEntry>>();
+ for (Map.Entry<String, List<ExportEntry>> entry : entries) {
+ this.entries.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Return the values as json string
+ *
+ * @return the JSON form
+ *
+ * @throws IOException mapping problems
+ */
+ public String asJson() throws IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+ String json = mapper.writeValueAsString(entries);
+ return json;
+ }
+
+ /**
+ * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's
+ * values
+ *
+ * @return the copy
+ */
+ public PublishedExports shallowCopy() {
+ PublishedExports that = new PublishedExports();
+ that.description = this.description;
+ that.updated = this.updated;
+ that.updatedTime = this.updatedTime;
+ return that;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("PublishedConfiguration{");
+ sb.append("description='").append(description).append('\'');
+ sb.append(" entries = ").append(entries.size());
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
new file mode 100644
index 0000000..67cb094
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Output a published configuration */
+public abstract class PublishedExportsOutputter {
+
+ protected final PublishedExports exports;
+
+ protected PublishedExportsOutputter(PublishedExports exports) {
+ this.exports = exports;
+ }
+
+ /**
+ * Create an outputter for the chosen format
+ *
+ * @param format format enumeration
+ * @param exports owning config
+ * @return the outputter
+ */
+
+ public static PublishedExportsOutputter createOutputter(ConfigFormat format,
+ PublishedExports exports) {
+ Preconditions.checkNotNull(exports);
+ switch (format) {
+ case JSON:
+ return new JsonOutputter(exports);
+ default:
+ throw new RuntimeException("Unsupported format :" + format);
+ }
+ }
+
+ public void save(File dest) throws IOException {
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(dest);
+ save(out);
+ out.close();
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(out);
+ }
+ }
+
+ /**
+ * Save the content. The default saves the asString() value to the output stream
+ *
+ * @param out output stream
+ * @throws IOException
+ */
+ public void save(OutputStream out) throws IOException {
+ IOUtils.write(asString(), out, Charsets.UTF_8);
+ }
+
+ /**
+ * Convert to a string
+ *
+ * @return the string form
+ * @throws IOException
+ */
+ public abstract String asString() throws IOException;
+
+ public static class JsonOutputter extends PublishedExportsOutputter {
+
+ public JsonOutputter(PublishedExports exports) {
+ super(exports);
+ }
+
+ @Override
+ public void save(File dest) throws IOException {
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+ }
+
+ @Override
+ public String asString() throws IOException {
+ return exports.asJson();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
new file mode 100644
index 0000000..339d3d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExportsSet {
+
+ private static final PatternValidator validator = new PatternValidator(
+ RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+
+ public Map<String, PublishedExports> exports = new HashMap<>();
+
+ public PublishedExportsSet() {
+ }
+
+ /**
+ * Put a name -it will be converted to lower case before insertion.
+ * Any existing entry will be overwritten (that includes an entry
+ * with a different case in the original name)
+ * @param name name of entry
+ * @param export published export
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void put(String name, PublishedExports export) {
+ String name1 = name.toLowerCase(Locale.ENGLISH);
+ validateName(name1);
+ exports.put(name1, export);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public static void validateName(String name) {
+ validator.validate(name);
+
+ }
+
+ public PublishedExports get(String name) {
+ return exports.get(name);
+ }
+
+ public boolean contains(String name) {
+ return exports.containsKey(name);
+ }
+
+ public int size() {
+ return exports.size();
+ }
+
+ public Set<String> keys() {
+ TreeSet<String> keys = new TreeSet<>();
+ keys.addAll(exports.keySet());
+ return keys;
+ }
+
+ public PublishedExportsSet shallowCopy() {
+ PublishedExportsSet that = new PublishedExportsSet();
+ for (Map.Entry<String, PublishedExports> entry : exports.entrySet()) {
+ that.put(entry.getKey(), entry.getValue().shallowCopy());
+ }
+ return that;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
new file mode 100644
index 0000000..120966f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class UriMap {
+
+ public Map<String, String> uris = new HashMap<>();
+
+ @JsonIgnore
+ public void put(String key, String value) {
+ uris.put(key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
new file mode 100644
index 0000000..13ad5c5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "classpath:org.apache.slider.management";
+
+ public static final String REGISTRY_REST_API =
+ "classpath:org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
+
+ public static final String AM_IPC_PROTOCOL =
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
+
+ public static final String WEB_UI = "http://";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
new file mode 100644
index 0000000..40fa217
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.retrieve;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.slider.client.rest.BaseRestClient;
+import org.apache.slider.core.restclient.HttpVerb;
+import org.apache.slider.core.restclient.UgiJerseyBinding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+
+/**
+ * Class to retrieve artifacts from the AM's web site. This sets up
+ * the redirection and security logic properly
+ */
+public class AMWebClient {
+
+
+ private final BaseRestClient restClient;
+ private static final Logger
+ log = LoggerFactory.getLogger(AMWebClient.class);
+
+
+ public AMWebClient(Configuration conf) {
+ UgiJerseyBinding binding = new UgiJerseyBinding(conf);
+
+ restClient = new BaseRestClient(binding.createJerseyClient());
+
+ }
+
+
+ private static URLConnectionClientHandler getUrlConnectionClientHandler() {
+ return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url)
+ throws IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
+ // is a redirect - are we changing schemes?
+ String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
+ String originalScheme = url.getProtocol();
+ String redirectScheme = URI.create(redirectLocation).getScheme();
+ if (!originalScheme.equals(redirectScheme)) {
+ // need to fake it out by doing redirect ourselves
+ log.info("Protocol change during redirect. Redirecting {} to URL {}",
+ url, redirectLocation);
+ URL redirectURL = new URL(redirectLocation);
+ connection = (HttpURLConnection) redirectURL.openConnection();
+ }
+ }
+ if (connection instanceof HttpsURLConnection) {
+ log.debug("Attempting to configure HTTPS connection using client "
+ + "configuration");
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ try {
+ HttpsURLConnection c = (HttpsURLConnection) connection;
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration());
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ } catch (Exception e) {
+ log.info("Unable to configure HTTPS connection from "
+ + "configuration. Using JDK properties.");
+ }
+
+ }
+ return connection;
+ }
+ });
+ }
+
+ public WebResource resource(String url) {
+ return restClient.resource(url);
+ }
+
+ public BaseRestClient getRestClient() {
+ return restClient;
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, Class<T> c) throws IOException {
+ return restClient.exec(method, resource, c);
+ }
+
+ /**
+ * Execute the operation. Failures are raised as IOException subclasses
+ * @param method method to execute
+ * @param resource resource to work against
+ * @param t type to work with
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
+ throws IOException {
+ return restClient.exec(method, resource, t);
+ }
+
+ /**
+ * Execute the GET operation. Failures are raised as IOException subclasses
+ * @param resource resource to work against
+ * @param c class to build
+ * @param <T> type expected
+ * @return an instance of the type T
+ * @throws IOException on any failure
+ */
+ public <T> T get(WebResource resource, Class<T> c) throws IOException {
+ return restClient.get(resource, c);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
new file mode 100644
index 0000000..b0eddb8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.retrieve;
+
+import com.beust.jcommander.Strings;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import static org.apache.slider.client.ClientRegistryBinder.*;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.ExceptionConverter;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Registry retriever.
+ * This hides the HTTP operations that take place to
+ * get the actual content
+ */
+public class RegistryRetriever extends AMWebClient {
+ private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class);
+
+ private final String externalConfigurationURL;
+ private final String internalConfigurationURL;
+ private final String externalExportsURL;
+ private final String internalExportsURL;
+
+ /**
+ * Retrieve from a service by locating the
+ * exported {@link CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API}
+ * and working off it.
+ *
+ * @param conf configuration to work from
+ * @param record service record
+ * @throws RegistryIOException the address type of the endpoint does
+ * not match that expected (i.e. not a list of URLs), missing endpoint...
+ */
+ public RegistryRetriever(Configuration conf, ServiceRecord record) throws RegistryIOException {
+ super(conf);
+ externalConfigurationURL = lookupRestAPI(record,
+ PUBLISHER_CONFIGURATIONS_API, true);
+ internalConfigurationURL = lookupRestAPI(record,
+ PUBLISHER_CONFIGURATIONS_API, false);
+ externalExportsURL = lookupRestAPI(record,
+ PUBLISHER_EXPORTS_API, true);
+ internalExportsURL = lookupRestAPI(record,
+ PUBLISHER_EXPORTS_API, false);
+ }
+
+ /**
+ * Does a bonded registry retriever have a configuration?
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return true if there is a URL to the configurations defined
+ */
+ public boolean hasConfigurations(boolean external) {
+ return !Strings.isStringEmpty(
+ external ? externalConfigurationURL : internalConfigurationURL);
+ }
+
+ /**
+ * Get the configurations of the registry
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return the configuration sets
+ */
+ public PublishedConfigSet getConfigurations(boolean external) throws
+ FileNotFoundException, IOException {
+
+ String confURL = getConfigurationURL(external);
+ WebResource webResource = resource(confURL);
+ return get(webResource, PublishedConfigSet.class);
+ }
+
+ protected String getConfigurationURL(boolean external) throws FileNotFoundException {
+ String confURL = external ? externalConfigurationURL: internalConfigurationURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL");
+ }
+ return confURL;
+ }
+
+ protected String getExportURL(boolean external) throws FileNotFoundException {
+ String confURL = external ? externalExportsURL: internalExportsURL;
+ if (Strings.isStringEmpty(confURL)) {
+ throw new FileNotFoundException("No configuration URL");
+ }
+ return confURL;
+ }
+
+ /**
+ * Get the configurations of the registry
+ * @param external flag to indicate that it is the external entries to fetch
+ * @return the configuration sets
+ */
+ public PublishedExportsSet getExports(boolean external) throws
+ FileNotFoundException, IOException {
+
+ String exportsUrl = getExportURL(external);
+ WebResource webResource = resource(exportsUrl);
+ return get(webResource, PublishedExportsSet.class);
+ }
+
+
+ /**
+ * Get a complete configuration, with all values
+ * @param configSet config set to ask for
+ * @param name name of the configuration
+ * @param external flag to indicate that it is an external configuration
+ * @return the retrieved config
+ * @throws IOException IO problems
+ */
+ public PublishedConfiguration retrieveConfiguration(PublishedConfigSet configSet,
+ String name,
+ boolean external) throws IOException {
+ String confURL = getConfigurationURL(external);
+ if (!configSet.contains(name)) {
+ throw new FileNotFoundException("Unknown configuration " + name);
+ }
+ confURL = SliderUtils.appendToURL(confURL, name);
+ WebResource webResource = resource(confURL);
+ return get(webResource, PublishedConfiguration.class);
+ }
+
+ /**
+ * Get a complete export, with all values
+ * @param exportSet
+ * @param name name of the configuration
+ * @param external flag to indicate that it is an external configuration
+ * @return the retrieved config
+ * @throws IOException IO problems
+ */
+ public PublishedExports retrieveExports(PublishedExportsSet exportSet,
+ String name,
+ boolean external) throws IOException {
+ if (!exportSet.contains(name)) {
+ throw new FileNotFoundException("Unknown export " + name);
+ }
+ String exportsURL = getExportURL(external);
+ exportsURL = SliderUtils.appendToURL(exportsURL, name);
+ return get(resource(exportsURL), PublishedExports.class);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("RegistryRetriever{");
+ sb.append("externalConfigurationURL='")
+ .append(externalConfigurationURL)
+ .append('\'');
+ sb.append(", internalConfigurationURL='")
+ .append(internalConfigurationURL)
+ .append('\'');
+ sb.append(", externalExportsURL='").append(externalExportsURL).append('\'');
+ sb.append(", internalExportsURL='").append(internalExportsURL).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
new file mode 100644
index 0000000..0266223
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A response for use as a return value from operations
+ */
+public class HttpOperationResponse {
+
+ public int responseCode;
+ public long lastModified;
+ public String contentType;
+ public byte[] data;
+ public Map<String, List<String>> headers;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
new file mode 100644
index 0000000..c040345
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+/**
+ * Http verbs with details on what they support in terms of submit and
+ * response bodies.
+ * <p>
+ * Those verbs which do support bodies in the response MAY NOT return it;
+ * if the response code is 204 then the answer is "no body", but the operation
+ * is considered a success.
+ */
+public enum HttpVerb {
+ GET("GET", false, true),
+ POST("POST", true, true),
+ PUT("PUT", true, true),
+ DELETE("DELETE", false, true),
+ HEAD("HEAD", false, false);
+
+ private final String verb;
+ private final boolean hasUploadBody;
+ private final boolean hasResponseBody;
+
+ HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
+ this.verb = verb;
+ this.hasUploadBody = hasUploadBody;
+ this.hasResponseBody = hasResponseBody;
+ }
+
+ public String getVerb() {
+ return verb;
+ }
+
+ public boolean hasUploadBody() {
+ return hasUploadBody;
+ }
+
+ public boolean hasResponseBody() {
+ return hasResponseBody;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
new file mode 100644
index 0000000..e453f52
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+/**
+ * Factory for URL connections; used behind the scenes in the Jersey integration.
+ * <p>
+ * Derived from the WebHDFS implementation.
+ */
+public class SliderURLConnectionFactory {
+ private static final Logger log =
+ LoggerFactory.getLogger(SliderURLConnectionFactory.class);
+
+ /**
+ * Timeout for socket connects and reads
+ */
+ public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
+ private final ConnectionConfigurator connConfigurator;
+
+ private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
+
+ /**
+ * Construct a new URLConnectionFactory based on the configuration. It will
+ * try to load SSL certificates when it is specified.
+ */
+ public static SliderURLConnectionFactory newInstance(Configuration conf) {
+ ConnectionConfigurator conn;
+ try {
+ conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ log.debug("Cannot load customized SSL configuration.", e);
+ conn = DEFAULT_CONFIGURATOR;
+ }
+ return new SliderURLConnectionFactory(conn);
+ }
+
+ private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
+ }
+
+ /**
+ * Create a new ConnectionConfigurator for SSL connections
+ */
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ SliderURLConnectionFactory.setupConnection(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * to open
+ * @return URLConnection
+ * @throws IOException
+ */
+ public URLConnection openConnection(URL url) throws IOException {
+ try {
+ return openConnection(url, false);
+ } catch (AuthenticationException e) {
+ // Unreachable
+ return null;
+ }
+ }
+
+ /**
+ * Opens a url with read and connect timeouts
+ *
+ * @param url
+ * URL to open
+ * @param isSpnego
+ * whether the url should be authenticated via SPNEGO
+ * @return URLConnection
+ * @throws IOException
+ * @throws AuthenticationException
+ */
+ public URLConnection openConnection(URL url, boolean isSpnego)
+ throws IOException, AuthenticationException {
+ if (isSpnego) {
+ log.debug("open AuthenticatedURL connection {}", url);
+ UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+ final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+ return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+ connConfigurator).openConnection(url, authToken);
+ } else {
+ log.debug("open URL connection {}", url);
+ URLConnection connection = url.openConnection();
+ if (connection instanceof HttpURLConnection) {
+ connConfigurator.configure((HttpURLConnection) connection);
+ }
+ return connection;
+ }
+ }
+
+ /**
+ * Sets connection parameters on the given URLConnection
+ *
+ * @param connection
+ * URLConnection to set
+ * @param socketTimeout
+ * the connection and read timeout of the connection.
+ */
+ private static void setupConnection(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ connection.setUseCaches(false);
+ if (connection instanceof HttpURLConnection) {
+ ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
+ }
+ }
+
+ private static class BasicConfigurator implements ConnectionConfigurator {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org