You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/07 21:10:17 UTC
[17/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported
from slider-core module. (jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
new file mode 100644
index 0000000..78bb8c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Component defined in master package metainfo.json
+ */
+public class Component extends AbstractComponent {
+
+ String category = CATEGORY_MASTER;
+ String publishConfig = Boolean.FALSE.toString();
+ String minInstanceCount = "0";
+ String maxInstanceCount;
+ String autoStartOnFailure = Boolean.FALSE.toString();
+ String appExports;
+ String compExports;
+ String type = TYPE_STANDARD;
+ List<ComponentExport> componentExports = new ArrayList<>();
+ List<DockerContainer> dockerContainers = new ArrayList<>();
+ List<ConfigFile> configFiles = new ArrayList<>();
+
+ public Component() {
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ public String getPublishConfig() {
+ return publishConfig;
+ }
+
+ public void setPublishConfig(String publishConfig) {
+ this.publishConfig = publishConfig;
+ }
+
+ public String getAutoStartOnFailure() {
+ return autoStartOnFailure;
+ }
+
+ public void setAutoStartOnFailure(String autoStartOnFailure) {
+ this.autoStartOnFailure = autoStartOnFailure;
+ }
+
+ public String getAppExports() {
+ return appExports;
+ }
+
+ public void setAppExports(String appExports) {
+ this.appExports = appExports;
+ }
+
+ public String getCompExports() {
+ return compExports;
+ }
+
+ public void setCompExports(String compExports) {
+ this.compExports = compExports;
+ }
+
+ public String getMinInstanceCount() {
+ return minInstanceCount;
+ }
+
+ @JsonProperty("dockerContainers")
+ public List<DockerContainer> getDockerContainers() {
+ return this.dockerContainers;
+ }
+
+ public Boolean getAutoStartOnFailureBoolean() {
+ if (SliderUtils.isUnset(getAutoStartOnFailure())) {
+ return Boolean.FALSE;
+ }
+
+ return Boolean.parseBoolean(getAutoStartOnFailure());
+ }
+
+ public int getMinInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(minInstanceCount)) {
+ return 0;
+ }
+
+ try {
+ return Integer.parseInt(minInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
+ }
+ }
+
+ public int getMaxInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(maxInstanceCount)) {
+ return Integer.MAX_VALUE;
+ }
+
+ try {
+ return Integer.parseInt(maxInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
+ }
+ }
+
+ public void setMinInstanceCount(String minInstanceCount) {
+ this.minInstanceCount = minInstanceCount;
+ }
+
+ public String getMaxInstanceCount() {
+ return maxInstanceCount;
+ }
+
+ public void setMaxInstanceCount(String maxInstanceCount) {
+ this.maxInstanceCount = maxInstanceCount;
+ }
+
+ public void addComponentExport(ComponentExport export) {
+ componentExports.add(export);
+ }
+
+ public List<ComponentExport> getComponentExports() {
+ return componentExports;
+ }
+
+ public Boolean getRequiresAutoRestart() {
+ return Boolean.parseBoolean(this.autoStartOnFailure);
+ }
+
+ public void addConfigFile(ConfigFile configFile) {
+ this.configFiles.add(configFile);
+ }
+
+ @JsonProperty("configFiles")
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append("\n\"name\": ").append(name);
+ sb.append(",\n\"category\": ").append(category);
+ sb.append(",\n\"commandScript\" :").append(commandScript);
+ for(DockerContainer dc : dockerContainers){
+ sb.append(",\n\"container\" :").append(dc.toString());
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "component");
+ Metainfo.checkNonNull(getCategory(), "category", "component");
+ if (!getCategory().equals(CATEGORY_MASTER)
+ && !getCategory().equals(CATEGORY_SLAVE)
+ && !getCategory().equals(CATEGORY_CLIENT)) {
+ throw new SliderException("Invalid category for the component " + getCategory());
+ }
+
+ Metainfo.checkNonNull(getType(), "type", "component");
+ if (!getType().equals(TYPE_DOCKER)
+ && !getType().equals(TYPE_STANDARD)) {
+ throw new SliderException("Invalid type for the component " + getType());
+ }
+
+ if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
+ if (getType().equals(TYPE_DOCKER)) {
+ throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
+ }
+
+ if (getCommands().size() > 0) {
+ throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
+ }
+ }
+
+ if (commandScript != null) {
+ commandScript.validate(version);
+ }
+
+ if (version.equals(Metainfo.VERSION_TWO_ONE)) {
+ for (ComponentCommand cc : getCommands()) {
+ cc.validate(version);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
new file mode 100644
index 0000000..52117c5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents the metadata associated with the application.
+ */
+public class ComponentCommand implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(ComponentCommand.class);
+
+
+ private String exec;
+ private String name = "START";
+ private String type = "SHELL";
+
+ /**
+ * Creator.
+ */
+ public ComponentCommand() {
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setExec(String exec) {
+ this.exec = exec;
+ }
+
+ public String getExec() {
+ return exec;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "componentCommand");
+
+ Metainfo.checkNonNull(getType(), "version", "application");
+ }
+
+ public static ComponentCommand getDefaultComponentCommand() {
+ ComponentCommand cc = new ComponentCommand();
+ cc.setExec("DEFAULT");
+ return cc;
+ }
+
+ public static ComponentCommand getDefaultComponentCommand(String commandName) {
+ ComponentCommand cc = new ComponentCommand();
+ cc.setExec("DEFAULT");
+ cc.setName(commandName);
+ return cc;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
new file mode 100644
index 0000000..a18854c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class ComponentExport {
+ String name;
+ String value;
+
+ public ComponentExport() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"name\": ").append(name);
+ sb.append(",\n\"value\": ").append(value);
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
new file mode 100644
index 0000000..855e5b6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+public class ComponentsInAddonPackage extends AbstractComponent {
+
+ @Override
+ public void validate(String version) throws SliderException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
new file mode 100644
index 0000000..cb47512
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class ConfigFile implements Validate {
+ String type;
+ String fileName;
+ String dictionaryName;
+
+ public ConfigFile() {
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public String getDictionaryName() {
+ return dictionaryName;
+ }
+
+ public void setDictionaryName(String dictionaryName) {
+ this.dictionaryName = dictionaryName;
+ }
+
+ public void validate(String version) throws SliderException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
new file mode 100644
index 0000000..46c8836
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Application default config
+ */
+public class DefaultConfig {
+ List<PropertyInfo> propertyInfos;
+
+ public DefaultConfig() {
+ propertyInfos = new ArrayList<PropertyInfo>();
+ }
+
+ public void addPropertyInfo(PropertyInfo propertyInfo) {
+ propertyInfos.add(propertyInfo);
+ }
+
+ public List<PropertyInfo> getPropertyInfos() {
+ return propertyInfos;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
new file mode 100644
index 0000000..e136775
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class DefaultConfigParser {
+
+ public DefaultConfig parse(InputStream configFileStream) throws IOException {
+ Digester digester = new Digester();
+ digester.setValidating(false);
+
+ digester.addObjectCreate("configuration", DefaultConfig.class);
+
+ digester.addObjectCreate("*/property", PropertyInfo.class);
+ digester.addBeanPropertySetter("*/property/name");
+ digester.addBeanPropertySetter("*/property/value");
+ digester.addBeanPropertySetter("*/property/description");
+ digester.addSetNext("*/property", "addPropertyInfo");
+
+ try {
+ return (DefaultConfig) digester.parse(configFileStream);
+ } catch (IOException e) {
+
+ } catch (SAXException e) {
+
+ } finally {
+ configFileStream.close();
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
new file mode 100644
index 0000000..4c61e7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a docker container
+ */
+public class DockerContainer implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(DockerContainer.class);
+
+ private String name;
+ private String image;
+ private String network;
+ private String useNetworkScript;
+ private String options;
+ private List<DockerContainerMount> mounts = new ArrayList<>();
+ private List<DockerContainerPort> ports = new ArrayList<>();
+ private String statusCommand;
+ private String startCommand;
+ private String commandPath;
+ private String additionalParam;
+ private String runPrivilegedContainer;
+ private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
+ private List<ConfigFile> configFiles = new ArrayList<>();
+
+ public DockerContainer() {
+ }
+
+ @JsonProperty("mounts")
+ public List<DockerContainerMount> getMounts() { return this.mounts; }
+
+ @JsonProperty("ports")
+ public List<DockerContainerPort> getPorts() {
+ return this.ports;
+ }
+
+ @JsonProperty("inputFiles")
+ public List<DockerContainerInputFile> getInputFiles() {
+ return this.inputFiles;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getImage() {
+ return image;
+ }
+
+ public void setImage(String image) {
+ this.image = image;
+ }
+
+ public String getNetwork() {
+ return network;
+ }
+
+ public void setNetwork(String network) {
+ this.network = network;
+ }
+
+ public String getUseNetworkScript() {
+ return useNetworkScript;
+ }
+
+ public void setUseNetworkScript(String useNetworkScript) {
+ this.useNetworkScript = useNetworkScript;
+ }
+
+ public String getOptions() {
+ return options;
+ }
+
+ public void setOptions(String options) {
+ this.options = options;
+ }
+
+ @Override
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "dockerContainer");
+ Metainfo.checkNonNull(getImage(), "image", "dockerContainer");
+ for (DockerContainerMount dcm : getMounts()) {
+ dcm.validate(version);
+ }
+ for (DockerContainerPort dcp : getPorts()) {
+ dcp.validate(version);
+ }
+ }
+
+ @JsonProperty("statusCommand")
+ public String getStatusCommand() {
+ return statusCommand;
+ }
+
+ @JsonProperty("statusCommand")
+ public void setStatusCommand(String statusCommand) {
+ this.statusCommand = statusCommand;
+ }
+
+ public String getCommandPath() {
+ return commandPath;
+ }
+
+ public void setCommandPath(String commandPath) {
+ this.commandPath = commandPath;
+ }
+
+ public String getAdditionalParam() {
+ return additionalParam;
+ }
+
+ public void setAdditionalParam(String additionalParam) {
+ this.additionalParam = additionalParam;
+ }
+
+ @JsonProperty("startCommand")
+ public String getStartCommand() {
+ return startCommand;
+ }
+
+ @JsonProperty("startCommand")
+ public void setStartCommand(String startCommand) {
+ this.startCommand = startCommand;
+ }
+
+ @JsonProperty("runPrivilegedContainer")
+ public String getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @JsonProperty("runPrivilegedContainer")
+ public void setRunPrivilegedContainer(String runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(List<ConfigFile> configFiles) {
+ this.configFiles = configFiles;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder("DockerContainer [name=")
+ .append(name).append(", image=").append(image).append(", options=")
+ .append(options).append(", mounts=").append(mounts).append(", ports=")
+ .append(ports).append(", statusCommand=").append(statusCommand)
+ .append(", commandPath=").append(commandPath)
+ .append(", additionalParam=").append(additionalParam)
+ .append(", inputFiles=").append(inputFiles).append(", startCommand=")
+ .append(startCommand).append(", runPriviledgedContainer=")
+ .append(runPrivilegedContainer).append(", configFiles=")
+ .append(configFiles).append("]");
+ return result.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
new file mode 100644
index 0000000..0faceb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DockerContainerInputFile {
+ protected static final Logger log = LoggerFactory
+ .getLogger(DockerContainerInputFile.class);
+
+ private String containerPath;
+ private String fileLocalPath;
+
+ public DockerContainerInputFile() {
+ }
+
+ public String getContainerMount() {
+ return containerPath;
+ }
+
+ public void setContainerMount(String containerMount) {
+ this.containerPath = containerMount;
+ }
+
+ public String getFileLocalPath() {
+ return fileLocalPath;
+ }
+
+ public void setFileLocalPath(String fileLocalPath) {
+ this.fileLocalPath = fileLocalPath;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
new file mode 100644
index 0000000..61f07f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a docker container mount
+ */
+public class DockerContainerMount implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(DockerContainerMount.class);
+
+
+ private String containerMount;
+ private String hostMount;
+
+ public DockerContainerMount() {
+ }
+
+ public String getContainerMount() {
+ return containerMount;
+ }
+
+ public void setContainerMount(String containerMount) {
+ this.containerMount = containerMount;
+ }
+
+ public String getHostMount() {
+ return hostMount;
+ }
+
+ public void setHostMount(String hostMount) {
+ this.hostMount = hostMount;
+ }
+
+ @Override
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount");
+ Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
new file mode 100644
index 0000000..0629d9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a docker container port
+ */
+public class DockerContainerPort implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(DockerContainerPort.class);
+
+
+ private String containerPort;
+ private String hostPort;
+
+ public DockerContainerPort() {
+ }
+
+ public String getContainerPort() {
+ return containerPort;
+ }
+
+ public void setContainerPort(String containerPort) {
+ this.containerPort = containerPort;
+ }
+
+ public String getHostPort() {
+ return hostPort;
+ }
+
+ public void setHostPort(String hostPort) {
+ this.hostPort = hostPort;
+ }
+
+ @Override
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort");
+ Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort");
+ }
+
+ @Override
+ public String toString() {
+ return "DockerContainerPort [containerPort=" + containerPort
+ + ", hostPort=" + hostPort + "]";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
new file mode 100644
index 0000000..5e0fb24
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class Export implements Validate {
+ String name;
+ String value;
+
+ public Export() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"name\": ").append(name);
+ sb.append(",\n\"value\": ").append(value);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "export");
+ Metainfo.checkNonNull(getValue(), "value", "export");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
new file mode 100644
index 0000000..3d9f34c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class ExportGroup implements Validate {
+ String name;
+ List<Export> exports;
+
+ public ExportGroup() {
+ exports = new ArrayList<Export>();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void addExport(Export export) {
+ exports.add(export);
+ }
+
+ public List<Export> getExports() {
+ return exports;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"name\": ").append(name);
+ sb.append(",\n\"exports\" : {");
+ for (Export export : exports) {
+ sb.append("\n").append(export);
+ }
+ sb.append("\n},");
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "exportGroup");
+ for(Export exp : getExports()) {
+ exp.validate(version);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
new file mode 100644
index 0000000..10c497f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Application metainfo uber class
+ */
+public class Metainfo {
+ protected static final Logger log =
+ LoggerFactory.getLogger(Metainfo.class);
+ public static String VERSION_TWO_ZERO = "2.0";
+ public static String VERSION_TWO_ONE = "2.1";
+
+ String schemaVersion;
+ ApplicationPackage applicationPackage;
+ Application application;
+
+ public String getSchemaVersion() {
+ return schemaVersion;
+ }
+
+ public void setSchemaVersion(String schemaVersion) {
+ this.schemaVersion = schemaVersion;
+ }
+
+ public ApplicationPackage getApplicationPackage() {
+ return applicationPackage;
+ }
+
+ public void setApplicationPackage(ApplicationPackage pkg) {
+ this.applicationPackage = pkg;
+ }
+
+ public Application getApplication() {
+ return application;
+ }
+
+ public void setApplication(Application application) {
+ this.application = application;
+ }
+
+ public Component getApplicationComponent(String roleGroup) {
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo");
+ } else {
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleGroup)) {
+ return component;
+ }
+ }
+ }
+ return null;
+ }
+
+ public List<ConfigFile> getComponentConfigFiles(String roleGroup) {
+ List<ConfigFile> componentConfigFiles = new ArrayList<>();
+ componentConfigFiles.addAll(application.getConfigFiles());
+ Component component = getApplicationComponent(roleGroup);
+ if (component != null) {
+ componentConfigFiles.addAll(component.getConfigFiles());
+ }
+ return componentConfigFiles;
+ }
+
+ public void validate() throws SliderException {
+ if (!VERSION_TWO_ONE.equals(schemaVersion) &&
+ !VERSION_TWO_ZERO.equals(schemaVersion)) {
+ throw new SliderException("Unsupported version " + getSchemaVersion());
+ }
+ if (application != null) {
+ application.validate(schemaVersion);
+ }
+ if (applicationPackage != null) {
+ applicationPackage.validate(schemaVersion);
+ }
+ }
+
+ public static void checkNonNull(String value, String field, String type) throws SliderException {
+ if (SliderUtils.isUnset(value)) {
+ throw new SliderException(type + "." + field + " cannot be null");
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Metainfo [schemaVersion=");
+ builder.append(schemaVersion);
+ builder.append(", applicationPackage=");
+ builder.append(applicationPackage);
+ builder.append(", application=");
+ builder.append(application);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
new file mode 100644
index 0000000..8b520eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.commons.digester.Digester;
+
+/**
+ *
+ */
+public class MetainfoParser extends AbstractMetainfoParser{
+
+ protected void composeSchema(Digester digester){
+ digester.addObjectCreate("metainfo", Metainfo.class);
+ digester.addBeanPropertySetter("metainfo/schemaVersion");
+
+ digester.addObjectCreate("*/application", Application.class);
+ digester.addBeanPropertySetter("*/application/name");
+ digester.addBeanPropertySetter("*/application/comment");
+ digester.addBeanPropertySetter("*/application/version");
+ digester.addBeanPropertySetter("*/application/exportedConfigs");
+
+ digester.addObjectCreate("*/commandOrder", CommandOrder.class);
+ digester.addBeanPropertySetter("*/commandOrder/command");
+ digester.addBeanPropertySetter("*/commandOrder/requires");
+ digester.addSetNext("*/commandOrder", "addCommandOrder");
+
+ digester.addObjectCreate("*/exportGroup", ExportGroup.class);
+ digester.addBeanPropertySetter("*/exportGroup/name");
+ digester.addObjectCreate("*/export", Export.class);
+ digester.addBeanPropertySetter("*/export/name");
+ digester.addBeanPropertySetter("*/export/value");
+ digester.addSetNext("*/export", "addExport");
+ digester.addSetNext("*/exportGroup", "addExportGroup");
+
+ digester.addObjectCreate("*/component", Component.class);
+ digester.addBeanPropertySetter("*/component/name");
+ digester.addBeanPropertySetter("*/component/category");
+ digester.addBeanPropertySetter("*/component/publishConfig");
+ digester.addBeanPropertySetter("*/component/minInstanceCount");
+ digester.addBeanPropertySetter("*/component/maxInstanceCount");
+ digester.addBeanPropertySetter("*/component/autoStartOnFailure");
+ digester.addBeanPropertySetter("*/component/appExports");
+ digester.addBeanPropertySetter("*/component/compExports");
+ digester.addObjectCreate("*/componentExport", ComponentExport.class);
+ digester.addBeanPropertySetter("*/componentExport/name");
+ digester.addBeanPropertySetter("*/componentExport/value");
+ digester.addSetNext("*/componentExport", "addComponentExport");
+ digester.addSetNext("*/component", "addComponent");
+
+ digester.addObjectCreate("*/commandScript", CommandScript.class);
+ digester.addBeanPropertySetter("*/commandScript/script");
+ digester.addBeanPropertySetter("*/commandScript/scriptType");
+ digester.addBeanPropertySetter("*/commandScript/timeout");
+ digester.addSetNext("*/commandScript", "addCommandScript");
+
+ digester.addObjectCreate("*/command", ComponentCommand.class);
+ digester.addBeanPropertySetter("*/command/exec");
+ digester.addBeanPropertySetter("*/command/name");
+ digester.addBeanPropertySetter("*/command/type");
+ digester.addSetNext("*/command", "addCommand");
+
+ digester.addObjectCreate("*/osSpecific", OSSpecific.class);
+ digester.addBeanPropertySetter("*/osSpecific/osType");
+ digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class);
+ digester.addBeanPropertySetter("*/osSpecific/packages/package/type");
+ digester.addBeanPropertySetter("*/osSpecific/packages/package/name");
+ digester.addSetNext("*/osSpecific/packages/package", "addOSPackage");
+ digester.addSetNext("*/osSpecific", "addOSSpecific");
+
+ digester.addObjectCreate("*/application/packages/package", Package.class);
+ digester.addBeanPropertySetter("*/application/packages/package/type");
+ digester.addBeanPropertySetter("*/application/packages/package/name");
+ digester.addSetNext("*/application/packages/package", "addPackage");
+
+ digester.addObjectCreate("*/configFile", ConfigFile.class);
+ digester.addBeanPropertySetter("*/configFile/type");
+ digester.addBeanPropertySetter("*/configFile/fileName");
+ digester.addBeanPropertySetter("*/configFile/dictionaryName");
+ digester.addSetNext("*/configFile", "addConfigFile");
+
+ digester.addSetRoot("*/application", "setApplication");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
new file mode 100644
index 0000000..32b4890
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ *
+ */
+public class OSPackage implements Validate {
+ String type;
+ String name;
+
+ public OSPackage() {
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "osPackage");
+ Metainfo.checkNonNull(getType(), "type", "osPackage");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
new file mode 100644
index 0000000..c06d498
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class OSSpecific implements Validate {
+ String osType;
+ List<OSPackage> packages;
+
+ public OSSpecific() {
+ packages = new ArrayList<OSPackage>();
+ }
+
+ public String getOsType() {
+ return osType;
+ }
+
+ public void setOsType(String osType) {
+ this.osType = osType;
+ }
+
+ public void addOSPackage(OSPackage osPackage) {
+ packages.add(osPackage);
+ }
+
+ public List<OSPackage> getPackages() {
+ return packages;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
+ for (OSPackage opkg : getPackages()) {
+ opkg.validate(version);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
new file mode 100644
index 0000000..b88f77d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents package description.
+ */
+public class Package implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(Package.class);
+
+
+ private String name;
+ private String type;
+
+ public Package() {
+ }
+
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "package");
+ Metainfo.checkNonNull(getType(), "type", "package");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
new file mode 100644
index 0000000..62ee0f5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ * Application config property info
+ */
+public class PropertyInfo {
+ String name;
+ String value;
+ String description;
+
+ public PropertyInfo() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
new file mode 100644
index 0000000..ef03dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ * Implementer provides a validate method
+ */
+public interface Validate {
+
+ public void validate(String version) throws SliderException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
new file mode 100644
index 0000000..dfd1373
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# todo list
+
+* Retry on failure
+ * Agent can toleate a configurable number of failures (e.g. 3) before giving up
+* Agent should separate out hostname and label that is received for registration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
new file mode 100644
index 0000000..e5430f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.launch.JavaCommandLineBuilder;
+import org.apache.slider.providers.AbstractClientProvider;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+
+/**
+ * handles the setup of the Slider AM.
+ * This keeps aspects of role, cluster validation and Clusterspec setup
+ * out of the core slider client
+ */
+public class SliderAMClientProvider extends AbstractClientProvider
+ implements SliderKeys {
+
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SliderAMClientProvider.class);
+ protected static final String NAME = "SliderAM";
+ public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT +
+ "slideram/instance/";
+ public static final String INTERNAL_JSON =
+ INSTANCE_RESOURCE_BASE + "internal.json";
+ public static final String APPCONF_JSON =
+ INSTANCE_RESOURCE_BASE + "appconf.json";
+ public static final String RESOURCES_JSON =
+ INSTANCE_RESOURCE_BASE + "resources.json";
+
+ public SliderAMClientProvider(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * List of roles
+ */
+ public static final List<ProviderRole> ROLES =
+ new ArrayList<ProviderRole>();
+
+ public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
+
+ public static final ProviderRole APPMASTER =
+ new ProviderRole(COMPONENT_AM, KEY_AM,
+ PlacementPolicy.EXCLUDE_FROM_FLEXING,
+ ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+ 0, "");
+
+ /**
+ * Initialize role list
+ */
+ static {
+ ROLES.add(APPMASTER);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public List<ProviderRole> getRoles() {
+ return ROLES;
+ }
+
+
+ @Override //Client
+ public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
+ String clustername,
+ Configuration configuration,
+ AggregateConf instanceDefinition,
+ Path clusterDirPath,
+ Path generatedConfDirPath,
+ boolean secure)
+ throws SliderException, IOException {
+
+ super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure);
+ //add a check for the directory being writeable by the current user
+ String
+ dataPath = instanceDefinition.getInternalOperations()
+ .getGlobalOptions()
+ .getMandatoryOption(
+ InternalKeys.INTERNAL_DATA_DIR_PATH);
+
+ Path path = new Path(dataPath);
+ sliderFileSystem.verifyDirectoryWriteAccess(path);
+ Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME);
+ sliderFileSystem.verifyDirectoryWriteAccess(historyPath);
+ }
+
+ /**
+ * Verify that an instance definition is considered valid by the provider
+ * @param instanceDefinition instance definition
+ * @throws SliderException if the configuration is not valid
+ */
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
+ SliderException {
+
+ super.validateInstanceDefinition(instanceDefinition, fs);
+
+ // make sure there is no negative entry in the instance count
+ Map<String, Map<String, String>> instanceMap =
+ instanceDefinition.getResources().components;
+ for (Map.Entry<String, Map<String, String>> entry : instanceMap.entrySet()) {
+ MapOperations mapOperations = new MapOperations(entry);
+ int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0);
+ if (instances < 0) {
+ throw new BadClusterStateException(
+ "Component %s has negative instance count: %d",
+ mapOperations.name,
+ instances);
+ }
+ }
+ }
+
+ /**
+ * The Slider AM sets up all the dependency JARs above slider.jar itself
+ * {@inheritDoc}
+ */
+ public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
+ Configuration serviceConf,
+ AbstractLauncher launcher,
+ AggregateConf instanceDescription,
+ Path snapshotConfDirPath,
+ Path generatedConfDirPath,
+ Configuration clientConfExtras,
+ String libdir,
+ Path tempPath, boolean miniClusterTestRun)
+ throws IOException, SliderException {
+
+ Map<String, LocalResource> providerResources = new HashMap<>();
+
+ ProviderUtils.addProviderJar(providerResources,
+ this,
+ SLIDER_JAR,
+ fileSystem,
+ tempPath,
+ libdir,
+ miniClusterTestRun);
+
+ String libDirProp =
+ System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ log.info("Loading all dependencies for AM.");
+ // If slider.tar.gz is available in hdfs use it, else upload all jars
+ Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
+ if (fileSystem.isFile(dependencyLibTarGzip)) {
+ SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
+ } else {
+ ProviderUtils.addAllDependencyJars(providerResources,
+ fileSystem,
+ tempPath,
+ libdir,
+ libDirProp);
+ }
+ addKeytabResourceIfNecessary(fileSystem,
+ instanceDescription,
+ providerResources);
+
+ launcher.addLocalResources(providerResources);
+
+ //also pick up all env variables from a map
+ launcher.copyEnvVars(
+ instanceDescription.getInternalOperations().getOrAddComponent(
+ SliderKeys.COMPONENT_AM));
+ }
+
+ /**
+ * If the cluster is secure, and an HDFS installed keytab is available for AM
+ * authentication, add this keytab as a local resource for the AM launch.
+ *
+ * @param fileSystem
+ * @param instanceDescription
+ * @param providerResources
+ * @throws IOException
+ * @throws BadConfigException if there's no keytab and it is explicitly required.
+ */
+ protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem,
+ AggregateConf instanceDescription,
+ Map<String, LocalResource> providerResources)
+ throws IOException, BadConfigException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ String keytabPathOnHost = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = instanceDescription.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName,
+ instanceDescription.getName());
+ if (fileSystem.getFileSystem().exists(keytabPath)) {
+ LocalResource keytabRes = fileSystem.createAmResource(keytabPath,
+ LocalResourceType.FILE);
+
+ providerResources.put(SliderKeys.KEYTAB_DIR + "/" +
+ amKeytabName, keytabRes);
+ } else {
+ log.warn("No keytab file was found at {}.", keytabPath);
+ if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ throw new BadConfigException("No keytab file was found at %s.", keytabPath);
+
+ } else {
+ log.warn("The AM will be "
+ + "started without a kerberos authenticated identity. "
+ + "The application is therefore not guaranteed to remain "
+ + "operational beyond 24 hours.");
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Update the AM resource with any local needs
+ * @param capability capability to update
+ */
+ public void prepareAMResourceRequirements(MapOperations sliderAM,
+ Resource capability) {
+ capability.setMemory(sliderAM.getOptionInt(
+ ResourceKeys.YARN_MEMORY,
+ capability.getMemory()));
+ capability.setVirtualCores(
+ sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores()));
+ }
+
+ /**
+ * Extract any JVM options from the cluster specification and
+ * add them to the command line
+ */
+ public void addJVMOptions(AggregateConf aggregateConf,
+ JavaCommandLineBuilder cmdLine)
+ throws BadConfigException {
+
+ MapOperations sliderAM =
+ aggregateConf.getAppConfOperations().getMandatoryComponent(
+ SliderKeys.COMPONENT_AM);
+ cmdLine.forceIPv4().headless();
+ String heap = sliderAM.getOption(RoleKeys.JVM_HEAP,
+ DEFAULT_JVM_HEAP);
+ cmdLine.setJVMHeap(heap);
+ String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, "");
+ if (SliderUtils.isSet(jvmopts)) {
+ cmdLine.add(jvmopts);
+ }
+ }
+
+
+ @Override
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf)
+ throws SliderException, IOException {
+ mergeTemplates(aggregateConf,
+ INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
+ );
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org