You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/05/13 02:54:01 UTC
[1/2] incubator-reef git commit: [REEF-300] HDInsight Job Submission
no longer works
Repository: incubator-reef
Updated Branches:
refs/heads/master d9f83715b -> 16f4e0737
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/StringEntry.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/StringEntry.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/StringEntry.java
new file mode 100644
index 0000000..a90dc33
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/StringEntry.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+/**
+ * An Entry with String Key and String Value in the Environment field
+ * and the ApplicationAcls field of an ApplicationSubmission.
+ * For detail information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class StringEntry {
+
+ private static final String STRING_ENTRY = "stringEntry";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private String key;
+ private String value;
+
+ public StringEntry(final String key, final String value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ @JsonProperty(Constants.KEY)
+ public String getKey() {
+ return this.key;
+ }
+
+ public void setKey(final String key) {
+ this.key = key;
+ }
+
+ @JsonProperty(Constants.VALUE)
+ public String getValue() {
+ return this.value;
+ }
+
+ public void setValue(final String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return STRING_ENTRY + objectString;
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final StringEntry that = (StringEntry) o;
+
+ return (this.key == that.key || (this.key != null && this.key.equals(that.key)))
+ && (this.value == that.value || (this.value != null && this.value.equals(that.value)));
+ }
+
+ @Override
+ public int hashCode() {
+ int result = this.key != null ? this.key.hashCode() : 0;
+ result = 31 * result + (this.value != null ? this.value.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/test/java/org/apache/reef/runtime/hdinsight/TestHDInsightRESTJsonSerialization.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/test/java/org/apache/reef/runtime/hdinsight/TestHDInsightRESTJsonSerialization.java b/lang/java/reef-runtime-hdinsight/src/test/java/org/apache/reef/runtime/hdinsight/TestHDInsightRESTJsonSerialization.java
new file mode 100644
index 0000000..56c6b4d
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/test/java/org/apache/reef/runtime/hdinsight/TestHDInsightRESTJsonSerialization.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight;
+
+import org.apache.reef.runtime.hdinsight.client.yarnrest.*;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * These tests apply to REST calls to the YARN Resource Manager
+ * for HDInsight 3.2, which corresponds to Hadoop version 2.6.0.
+ * Details are documented at:
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class TestHDInsightRESTJsonSerialization {
+
+ @Test
+ public void TestSerializeApplicationSubmission() throws IOException {
+ // Define expected values
+ final Map<String, LocalResource> localResourceMap = new HashMap<>();
+ final String archiveKey = "archive";
+ final String fileKey = "file";
+
+ // Create submission object
+ LocalResource archiveResource = new LocalResource()
+ .setResource("archiveResourceLocation").setSize(100).setTimestamp(200)
+ .setType(LocalResource.TYPE_ARCHIVE).setVisibility(LocalResource.VISIBILITY_PRIVATE);
+ localResourceMap.put(archiveKey, archiveResource);
+
+ LocalResource fileResource = new LocalResource()
+ .setResource("fileResourceLocation").setSize(300).setTimestamp(400)
+ .setType(LocalResource.TYPE_FILE).setVisibility(LocalResource.VISIBILITY_APPLICATION);
+ localResourceMap.put(fileKey, fileResource);
+
+ Credentials creds = new Credentials().addSecret("secretKey", "secretVal").addToken("tokKey", "tokVal");
+ AmContainerSpec containerSpec = new AmContainerSpec()
+ .setCommand("submission command")
+ .addLocalResource(archiveKey, archiveResource)
+ .addLocalResource(fileKey, fileResource)
+ .addEnvironment("envKey", "envVal")
+ .addApplicationAcl("aclKey", "aclVal")
+ .addServiceData("sdKey", "sdVal")
+ .setCredentials(creds);
+ Resource resource = new Resource().setMemory(500).setvCores(600);
+ ApplicationSubmission submission = new ApplicationSubmission()
+ .setApplicationType(ApplicationSubmission.DEFAULT_APPLICATION_TYPE)
+ .setMaxAppAttempts(ApplicationSubmission.DEFAULT_MAX_APP_ATTEMPTS)
+ .setKeepContainers(ApplicationSubmission.DEFAULT_KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS)
+ .setQueue(ApplicationSubmission.DEFAULT_QUEUE)
+ .setPriority(ApplicationSubmission.DEFAULT_PRIORITY)
+ .setUnmanagedAM(ApplicationSubmission.DEFAULT_UNMANAGED_AM)
+ .setAmContainerSpec(containerSpec).setApplicationId("appId").setApplicationName("name")
+ .setResource(resource);
+
+ // Json validation
+ ObjectMapper mapper = new ObjectMapper();
+ StringWriter writer = new StringWriter();
+ mapper.writeValue(writer, submission);
+ String jsonStr = writer.toString();
+ JsonNode rootJsonNode = mapper.readTree(jsonStr);
+ Assert.assertEquals(rootJsonNode.get(Constants.APPLICATION_ID).asText(), submission.getApplicationId());
+ Assert.assertEquals(rootJsonNode.get(Constants.APPLICATION_NAME).asText(), submission.getApplicationName());
+ Assert.assertEquals(rootJsonNode.get(Constants.MAX_APP_ATTEMPTS).asInt(), submission.getMaxAppAttempts());
+ Assert.assertEquals(rootJsonNode.get(Constants.KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS).asBoolean(), submission.isKeepContainers());
+ Assert.assertEquals(rootJsonNode.get(Constants.QUEUE).asText(), submission.getQueue());
+ Assert.assertEquals(rootJsonNode.get(Constants.PRIORITY).asInt(), submission.getPriority());
+ Assert.assertEquals(rootJsonNode.get(Constants.UNMANAGED_AM).asBoolean(), submission.isUnmanagedAM());
+
+ JsonNode resourceNode = rootJsonNode.get(Constants.RESOURCE);
+ Assert.assertNotNull(resourceNode);
+ Assert.assertEquals(resourceNode.get(Constants.MEMORY).asInt(), resource.getMemory());
+ Assert.assertEquals(resourceNode.get(Constants.VCORES).asInt(), resource.getvCores());
+
+ JsonNode amSpecNode = rootJsonNode.get(Constants.AM_CONTAINER_SPEC);
+ Assert.assertNotNull(amSpecNode);
+ Assert.assertEquals(amSpecNode.get(Constants.COMMANDS).get(Constants.COMMAND).asText(), containerSpec.getCommands().getCommand());
+ JsonNode locResourcesNode = amSpecNode.get(Constants.LOCAL_RESOURCES).get(Constants.ENTRY);
+ Assert.assertTrue(locResourcesNode.isArray());
+ for (final JsonNode localResourceKVNode : locResourcesNode) {
+ String localResourceKey = localResourceKVNode.get(Constants.KEY).asText();
+ Assert.assertTrue(localResourceMap.containsKey(localResourceKey));
+ LocalResource localResourceFromMap = localResourceMap.get(localResourceKey);
+ JsonNode localResourceNode = localResourceKVNode.get(Constants.VALUE);
+ Assert.assertEquals(localResourceNode.get(Constants.RESOURCE).asText(), localResourceFromMap.getResource());
+ Assert.assertEquals(localResourceNode.get(Constants.SIZE).asLong(), localResourceFromMap.getSize());
+ Assert.assertEquals(localResourceNode.get(Constants.TIMESTAMP).asLong(), localResourceFromMap.getTimestamp());
+ Assert.assertEquals(localResourceNode.get(Constants.TYPE).asText(), localResourceFromMap.getType());
+ Assert.assertEquals(localResourceNode.get(Constants.VISIBILITY).asText(), localResourceFromMap.getVisibility());
+ localResourceMap.remove(localResourceKey);
+ }
+
+ Assert.assertTrue(localResourceMap.isEmpty());
+
+ JsonNode credsNode = amSpecNode.get(Constants.CREDENTIALS);
+ Assert.assertNotNull(credsNode);
+ JsonNode toksNode = credsNode.get(Constants.TOKENS).get(Constants.ENTRY);
+ Assert.assertNotNull(toksNode);
+ Assert.assertTrue(toksNode.isArray());
+ for (final JsonNode tokKVNode : toksNode) {
+ StringEntry tokenEntry = containerSpec.getCredentials().getTokens().get(Constants.ENTRY).get(0);
+ Assert.assertEquals(tokKVNode.get(Constants.KEY).asText(), tokenEntry.getKey());
+ Assert.assertEquals(tokKVNode.get(Constants.VALUE).asText(), tokenEntry.getValue());
+ }
+ JsonNode secretsNode = credsNode.get(Constants.SECRETS).get(Constants.ENTRY);
+ Assert.assertNotNull(secretsNode);
+ Assert.assertTrue(secretsNode.isArray());
+ for (final JsonNode secretsKVNode : secretsNode) {
+ StringEntry secretsEntry = containerSpec.getCredentials().getSecrets().get(Constants.ENTRY).get(0);
+ Assert.assertEquals(secretsKVNode.get(Constants.KEY).asText(), secretsEntry.getKey());
+ Assert.assertEquals(secretsKVNode.get(Constants.VALUE).asText(), secretsEntry.getValue());
+ }
+
+ JsonNode envsNode = amSpecNode.get(Constants.ENVIRONMENT).get(Constants.ENTRY);
+ Assert.assertNotNull(envsNode);
+ Assert.assertTrue(envsNode.isArray());
+ for (final JsonNode envsKVNode : envsNode) {
+ StringEntry envEntry = containerSpec.getEnvironment().get(Constants.ENTRY).get(0);
+ Assert.assertEquals(envsKVNode.get(Constants.KEY).asText(), envEntry.getKey());
+ Assert.assertEquals(envsKVNode.get(Constants.VALUE).asText(), envEntry.getValue());
+ }
+
+ JsonNode aclsNode = amSpecNode.get(Constants.APPLICATION_ACLS).get(Constants.ENTRY);
+ Assert.assertNotNull(aclsNode);
+ Assert.assertTrue(aclsNode.isArray());
+ for (final JsonNode aclsKVNode : aclsNode) {
+ StringEntry aclsEntry = containerSpec.getApplicationAcls().get(Constants.ENTRY).get(0);
+ Assert.assertEquals(aclsKVNode.get(Constants.KEY).asText(), aclsEntry.getKey());
+ Assert.assertEquals(aclsKVNode.get(Constants.VALUE).asText(), aclsEntry.getValue());
+ }
+
+ JsonNode sdatasNode = amSpecNode.get(Constants.SERVICE_DATA).get(Constants.ENTRY);
+ Assert.assertNotNull(sdatasNode);
+ Assert.assertTrue(sdatasNode.isArray());
+ for (final JsonNode sdataKVNode : sdatasNode) {
+ StringEntry sdataEntry = containerSpec.getServiceData().get(Constants.ENTRY).get(0);
+ Assert.assertEquals(sdataKVNode.get(Constants.KEY).asText(), sdataEntry.getKey());
+ Assert.assertEquals(sdataKVNode.get(Constants.VALUE).asText(), sdataEntry.getValue());
+ }
+ }
+
+ @Test
+ public void TestDeserializeGetApplicationId() throws IOException {
+ final String appIdStr = "application_1404198295326_0003";
+ final int memory = 8192;
+ final int vCores = 32;
+
+ final String getAppIdBody = "{\n" +
+ " \"application-id\":\"" + appIdStr + "\",\n" +
+ " \"maximum-resource-capability\":\n" +
+ " {\n" +
+ " \"memory\":" + memory + ",\n" +
+ " \"vCores\":" + vCores + "\n" +
+ " }\n" +
+ "}";
+
+ ApplicationID appId = new ObjectMapper().readValue(getAppIdBody, ApplicationID.class);
+ Assert.assertEquals(appId.getApplicationId(), appIdStr);
+ Assert.assertEquals(appId.getResource().getMemory(), memory);
+ Assert.assertEquals(appId.getResource().getvCores(), vCores);
+ }
+
+ @Test
+ public void TestDeserializeGetApplication() throws IOException {
+ final long finishedTime = 1326824991300L;
+ final String amContainerLogs = "http://host.domain.com:8042/node/containerlogs/container_1326821518301_0005_01_000001";
+ final String trackingUI = "History";
+ final String state = "FINISHED";
+ final String user = "user1";
+ final String appId = "application_1326821518301_0005";
+ final String clusterId = "1326821518301";
+ final String finalStatus = "SUCCEEDED";
+ final String amHostHttpAddress = "host.domain.com:8042";
+ final String progress = "100";
+ final String name = "Sleep job";
+ final String applicationType = "Yarn";
+ final long startedTime = 1326824544552L;
+ final long elapsedTime = 446748L;
+ final String diagnostics = "";
+ final String trackingUrl = "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5";
+ final String queue = "a1";
+ final long memorySeconds = 151730L;
+ final long vcoreSeconds = 103L;
+
+ final String getAppBody = "{\n" +
+ " \"app\" : {\n" +
+ " \"finishedTime\" : " + finishedTime + ",\n" +
+ " \"amContainerLogs\" : \"" + amContainerLogs + "\",\n" +
+ " \"trackingUI\" : \"" + trackingUI + "\",\n" +
+ " \"state\" : \"" + state + "\",\n" +
+ " \"user\" : \"" + user + "\",\n" +
+ " \"id\" : \"" + appId + "\",\n" +
+ " \"clusterId\" : " + clusterId + ",\n" +
+ " \"finalStatus\" : \"" + finalStatus + "\",\n" +
+ " \"amHostHttpAddress\" : \"" + amHostHttpAddress + "\",\n" +
+ " \"progress\" : " + progress + ",\n" +
+ " \"name\" : \"" + name + "\",\n" +
+ " \"applicationType\" : \"" + applicationType + "\",\n" +
+ " \"startedTime\" : " + startedTime + ",\n" +
+ " \"elapsedTime\" : " + elapsedTime + ",\n" +
+ " \"diagnostics\" : \"" + diagnostics + "\",\n" +
+ " \"trackingUrl\" : \"" + trackingUrl + "\",\n" +
+ " \"queue\" : \"" + queue + "\",\n" +
+ " \"memorySeconds\" : " + memorySeconds + ",\n" +
+ " \"vcoreSeconds\" : " + vcoreSeconds + "\n" +
+ " }" +
+ "}";
+
+ ApplicationResponse appResponse = new ObjectMapper().readValue(getAppBody, ApplicationResponse.class);
+ ApplicationState appState = appResponse.getApplicationState();
+ Assert.assertEquals(appState.getFinishedTime(), finishedTime);
+ Assert.assertEquals(appState.getAmContainerLogs(), amContainerLogs);
+ Assert.assertEquals(appState.getTrackingUI(), trackingUI);
+ Assert.assertEquals(appState.getState(), state);
+ Assert.assertEquals(appState.getUser(), user);
+ Assert.assertEquals(appState.getId(), appId);
+ Assert.assertEquals(appState.getClusterId(), clusterId);
+ Assert.assertEquals(appState.getFinalStatus(), finalStatus);
+ Assert.assertEquals(appState.getAmHostHttpAddress(), amHostHttpAddress);
+ Assert.assertEquals(appState.getProgress(), progress);
+ Assert.assertEquals(appState.getName(), name);
+ Assert.assertEquals(appState.getApplicationType(), applicationType);
+ Assert.assertEquals(appState.getStartedTime(), startedTime);
+ Assert.assertEquals(appState.getElapsedTime(), elapsedTime);
+ Assert.assertEquals(appState.getDiagnostics(), diagnostics);
+ Assert.assertEquals(appState.getTrackingUrl(), trackingUrl);
+ Assert.assertEquals(appState.getQueue(), queue);
+ Assert.assertEquals(appState.getMemorySeconds(), memorySeconds);
+ Assert.assertEquals(appState.getVCoreSeconds(), vcoreSeconds);
+ }
+
+ @Test
+ public void TestDeserializeListApplications() throws IOException {
+ final String listAppsBody = "{\n" +
+ " \"apps\":\n" +
+ " {\n" +
+ " \"app\":\n" +
+ " [\n" +
+ " {\n" +
+ " \"finishedTime\" : 1326815598530,\n" +
+ " \"amContainerLogs\" : \"http://host.domain.com:8042/node/containerlogs/container_1326815542473_0001_01_000001\",\n" +
+ " \"trackingUI\" : \"History\",\n" +
+ " \"state\" : \"FINISHED\",\n" +
+ " \"user\" : \"user1\",\n" +
+ " \"id\" : \"application_1326815542473_0001\",\n" +
+ " \"clusterId\" : 1326815542473,\n" +
+ " \"finalStatus\" : \"SUCCEEDED\",\n" +
+ " \"amHostHttpAddress\" : \"host.domain.com:8042\",\n" +
+ " \"progress\" : 100,\n" +
+ " \"name\" : \"word count\",\n" +
+ " \"startedTime\" : 1326815573334,\n" +
+ " \"elapsedTime\" : 25196,\n" +
+ " \"diagnostics\" : \"\",\n" +
+ " \"trackingUrl\" : \"http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job/job_1326815542473_1_1\",\n" +
+ " \"queue\" : \"default\",\n" +
+ " \"allocatedMB\" : 0,\n" +
+ " \"allocatedVCores\" : 0,\n" +
+ " \"runningContainers\" : 0,\n" +
+ " \"memorySeconds\" : 151730,\n" +
+ " \"vcoreSeconds\" : 103\n" +
+ " },\n" +
+ " {\n" +
+ " \"finishedTime\" : 1326815789546,\n" +
+ " \"amContainerLogs\" : \"http://host.domain.com:8042/node/containerlogs/container_1326815542473_0002_01_000001\",\n" +
+ " \"trackingUI\" : \"History\",\n" +
+ " \"state\" : \"FINISHED\",\n" +
+ " \"user\" : \"user1\",\n" +
+ " \"id\" : \"application_1326815542473_0002\",\n" +
+ " \"clusterId\" : 1326815542473,\n" +
+ " \"finalStatus\" : \"SUCCEEDED\",\n" +
+ " \"amHostHttpAddress\" : \"host.domain.com:8042\",\n" +
+ " \"progress\" : 100,\n" +
+ " \"name\" : \"Sleep job\",\n" +
+ " \"startedTime\" : 1326815641380,\n" +
+ " \"elapsedTime\" : 148166,\n" +
+ " \"diagnostics\" : \"\",\n" +
+ " \"trackingUrl\" : \"http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2\",\n" +
+ " \"queue\" : \"default\",\n" +
+ " \"allocatedMB\" : 0,\n" +
+ " \"allocatedVCores\" : 0,\n" +
+ " \"runningContainers\" : 1,\n" +
+ " \"memorySeconds\" : 640064,\n" +
+ " \"vcoreSeconds\" : 442\n" +
+ " } \n" +
+ " ]\n" +
+ " }\n" +
+ "}";
+
+ ListApplicationResponse listAppsResponse = new ObjectMapper().readValue(listAppsBody, ListApplicationResponse.class);
+ Assert.assertTrue(listAppsResponse.getApps().containsKey(Constants.APP));
+ Assert.assertEquals(listAppsResponse.getApplicationStates().size(), 2);
+ for (ApplicationState state : listAppsResponse.getApplicationStates()) {
+ Assert.assertNotNull(state);
+ }
+ }
+}
\ No newline at end of file
[2/2] incubator-reef git commit: [REEF-300] HDInsight Job Submission
no longer works
Posted by we...@apache.org.
[REEF-300] HDInsight Job Submission no longer works
This addressed the issue by
* Updating Java objects to conform to REST API specified by
YARN 2.6.
* Fixing TANG configuration merge issues in
HDInsightRuntimeConfiguration
* Adding tests for serialization/deserialization for JSON objects
using examples from
https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
JIRA:
[REEF-300](https://issues.apache.org/jira/browse/REEF-300)
Pull Request:
This closes #171
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/16f4e073
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/16f4e073
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/16f4e073
Branch: refs/heads/master
Commit: 16f4e0737084b0c0f8f917c6a489f656ef40e627
Parents: d9f8371
Author: Andrew Chung <af...@gmail.com>
Authored: Sat Apr 4 12:52:22 2015 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Tue May 12 17:49:45 2015 -0700
----------------------------------------------------------------------
lang/java/reef-runtime-hdinsight/pom.xml | 6 +-
.../reef/runtime/hdinsight/cli/HDICLI.java | 3 +-
.../runtime/hdinsight/client/AzureUploader.java | 18 +-
.../client/HDInsightJobSubmissionHandler.java | 22 +-
.../client/HDInsightRuntimeConfiguration.java | 5 +-
.../UnsafeHDInsightRuntimeConfiguration.java | 5 +-
...safeHDInsightRuntimeConfigurationStatic.java | 4 +-
.../client/yarnrest/AmContainerSpec.java | 175 ++++++++++
.../client/yarnrest/ApplicationID.java | 41 ++-
.../client/yarnrest/ApplicationResponse.java | 46 ++-
.../client/yarnrest/ApplicationState.java | 223 ++++++++-----
.../client/yarnrest/ApplicationSubmission.java | 121 ++++---
.../hdinsight/client/yarnrest/Commands.java | 66 ++++
.../hdinsight/client/yarnrest/Constants.java | 84 +++++
.../client/yarnrest/ContainerInfo.java | 125 --------
.../hdinsight/client/yarnrest/Credentials.java | 101 ++++++
.../client/yarnrest/EnvironmentEntry.java | 76 -----
.../hdinsight/client/yarnrest/FileResource.java | 89 ------
.../client/yarnrest/HDInsightInstance.java | 50 ++-
.../yarnrest/ListApplicationResponse.java | 73 +++++
.../client/yarnrest/LocalResource.java | 113 +++++++
.../client/yarnrest/LocalResourcesEntry.java | 40 ++-
.../hdinsight/client/yarnrest/Resource.java | 40 ++-
.../hdinsight/client/yarnrest/StringEntry.java | 95 ++++++
.../TestHDInsightRESTJsonSerialization.java | 318 +++++++++++++++++++
25 files changed, 1442 insertions(+), 497 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/pom.xml b/lang/java/reef-runtime-hdinsight/pom.xml
index b9cb837..e04d492 100644
--- a/lang/java/reef-runtime-hdinsight/pom.xml
+++ b/lang/java/reef-runtime-hdinsight/pom.xml
@@ -77,7 +77,11 @@ under the License.
<artifactId>hadoop-yarn-client</artifactId>
<scope>provided</scope>
</dependency>
-
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/cli/HDICLI.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/cli/HDICLI.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/cli/HDICLI.java
index 93b3933..37631d7 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/cli/HDICLI.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/cli/HDICLI.java
@@ -103,8 +103,9 @@ public final class HDICLI {
* Kills the application with the given id.
*
* @param applicationId
+ * @throws IOException
*/
- private void kill(final String applicationId) {
+ private void kill(final String applicationId) throws IOException {
LOG.log(Level.INFO, "Killing application [{0}]", applicationId);
this.hdInsightInstance.killApplication(applicationId);
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/AzureUploader.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/AzureUploader.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/AzureUploader.java
index 055ea93..03249ce 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/AzureUploader.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/AzureUploader.java
@@ -26,7 +26,7 @@ import com.microsoft.windowsazure.storage.blob.CloudBlobContainer;
import com.microsoft.windowsazure.storage.blob.CloudBlockBlob;
import org.apache.reef.annotations.audience.ClientSide;
import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.runtime.hdinsight.client.yarnrest.FileResource;
+import org.apache.reef.runtime.hdinsight.client.yarnrest.LocalResource;
import org.apache.reef.runtime.hdinsight.parameters.AzureStorageAccountContainerName;
import org.apache.reef.runtime.hdinsight.parameters.AzureStorageAccountKey;
import org.apache.reef.runtime.hdinsight.parameters.AzureStorageAccountName;
@@ -100,7 +100,7 @@ final class AzureUploader {
}
}
- public FileResource uploadFile(final File file) throws IOException {
+ public LocalResource uploadFile(final File file) throws IOException {
final String destination = this.jobFolderName + "/" + file.getName();
LOG.log(Level.INFO, "Uploading [{0}] to [{1}]", new Object[]{file, destination});
@@ -121,14 +121,14 @@ final class AzureUploader {
LOG.log(Level.FINE, "Uploaded to: {0}",
jobJarBlob.getStorageUri().getPrimaryUri());
- // Assemble the FileResource
+ // Assemble the LocalResource
final BlobProperties blobProperties = jobJarBlob.getProperties();
- return new FileResource()
- .setType(FileResource.TYPE_ARCHIVE)
- .setVisibility(FileResource.VISIBILITY_APPLICATION)
- .setSize(String.valueOf(blobProperties.getLength()))
- .setTimestamp(String.valueOf(blobProperties.getLastModified().getTime()))
- .setUrl(getFileSystemURL(jobJarBlob));
+ return new LocalResource()
+ .setType(LocalResource.TYPE_ARCHIVE)
+ .setVisibility(LocalResource.VISIBILITY_APPLICATION)
+ .setSize(blobProperties.getLength())
+ .setTimestamp(blobProperties.getLastModified().getTime())
+ .setResource(getFileSystemURL(jobJarBlob));
} catch (final URISyntaxException | StorageException e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
index dc56abf..14ca392 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightJobSubmissionHandler.java
@@ -85,35 +85,35 @@ public final class HDInsightJobSubmissionHandler implements JobSubmissionHandler
LOG.log(Level.FINE, "Requesting Application ID from HDInsight.");
final ApplicationID applicationID = this.hdInsightInstance.getApplicationID();
- LOG.log(Level.INFO, "Submitting application {0} to YARN.", applicationID.getId());
+ LOG.log(Level.INFO, "Submitting application {0} to YARN.", applicationID.getApplicationId());
LOG.log(Level.FINE, "Creating a job folder on Azure.");
- final String jobFolderURL = this.uploader.createJobFolder(applicationID.getId());
+ final String jobFolderURL = this.uploader.createJobFolder(applicationID.getApplicationId());
LOG.log(Level.FINE, "Assembling Configuration for the Driver.");
final Configuration driverConfiguration =
- makeDriverConfiguration(jobSubmissionEvent, applicationID.getId(), jobFolderURL);
+ makeDriverConfiguration(jobSubmissionEvent, applicationID.getApplicationId(), jobFolderURL);
LOG.log(Level.FINE, "Making Job JAR.");
final File jobSubmissionJarFile =
this.jobJarMaker.createJobSubmissionJAR(jobSubmissionEvent, driverConfiguration);
LOG.log(Level.FINE, "Uploading Job JAR to Azure.");
- final FileResource uploadedFile = this.uploader.uploadFile(jobSubmissionJarFile);
+ final LocalResource uploadedFile = this.uploader.uploadFile(jobSubmissionJarFile);
LOG.log(Level.FINE, "Assembling application submission.");
final String command = getCommandString(jobSubmissionEvent);
final ApplicationSubmission applicationSubmission = new ApplicationSubmission()
- .setApplicationId(applicationID.getId())
+ .setApplicationId(applicationID.getApplicationId())
.setApplicationName(jobSubmissionEvent.getIdentifier())
.setResource(getResource(jobSubmissionEvent))
- .setContainerInfo(new ContainerInfo()
- .addFileResource(this.filenames.getREEFFolderName(), uploadedFile)
- .addCommand(command));
+ .setAmContainerSpec(new AmContainerSpec()
+ .addLocalResource(this.filenames.getREEFFolderName(), uploadedFile)
+ .setCommand(command));
this.hdInsightInstance.submitApplication(applicationSubmission);
- LOG.log(Level.INFO, "Submitted application to HDInsight. The application id is: {0}", applicationID.getId());
+ LOG.log(Level.INFO, "Submitted application to HDInsight. The application id is: {0}", applicationID.getApplicationId());
} catch (final IOException ex) {
LOG.log(Level.SEVERE, "Error submitting HDInsight request", ex);
@@ -128,8 +128,8 @@ public final class HDInsightJobSubmissionHandler implements JobSubmissionHandler
final JobSubmissionEvent jobSubmissionEvent) {
return new Resource()
- .setMemory(String.valueOf(jobSubmissionEvent.getDriverMemory().get()))
- .setvCores("1");
+ .setMemory(jobSubmissionEvent.getDriverMemory().get())
+ .setvCores(1);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
index 22e3d1d..2047360 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/HDInsightRuntimeConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.reef.runtime.hdinsight.client;
import org.apache.reef.runtime.hdinsight.parameters.*;
import org.apache.reef.tang.Configuration;
-import org.apache.reef.tang.Configurations;
import org.apache.reef.tang.formats.AvroConfigurationSerializer;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
@@ -87,9 +86,7 @@ public final class HDInsightRuntimeConfiguration extends ConfigurationModuleBuil
* @throws IOException if the file can't be read
*/
public static Configuration fromTextFile(final File file) throws IOException {
- final Configuration loaded = new AvroConfigurationSerializer().fromTextFile(file);
- final Configuration staticConfiguration = HDInsightRuntimeConfigurationStatic.CONF.build();
- return Configurations.merge(loaded, staticConfiguration);
+ return new AvroConfigurationSerializer().fromTextFile(file);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
index a96f5e5..2a914c8 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.reef.runtime.hdinsight.client;
import org.apache.reef.runtime.hdinsight.parameters.*;
import org.apache.reef.tang.Configuration;
-import org.apache.reef.tang.Configurations;
import org.apache.reef.tang.formats.AvroConfigurationSerializer;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
@@ -82,9 +81,7 @@ public final class UnsafeHDInsightRuntimeConfiguration extends ConfigurationModu
* @throws java.io.IOException if the file can't be read
*/
public static Configuration fromTextFile(final File file) throws IOException {
- final Configuration loaded = new AvroConfigurationSerializer().fromTextFile(file);
- final Configuration staticConfiguration = UnsafeHDInsightRuntimeConfigurationStatic.CONF.build();
- return Configurations.merge(loaded, staticConfiguration);
+ return new AvroConfigurationSerializer().fromTextFile(file);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
index cf7686d..0779a3a 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/UnsafeHDInsightRuntimeConfigurationStatic.java
@@ -26,8 +26,8 @@ import org.apache.reef.runtime.common.client.RunningJobImpl;
import org.apache.reef.runtime.common.client.api.JobSubmissionHandler;
import org.apache.reef.runtime.common.files.RuntimeClasspathProvider;
import org.apache.reef.runtime.common.launch.REEFMessageCodec;
+import org.apache.reef.runtime.hdinsight.HDInsightClasspathProvider;
import org.apache.reef.runtime.hdinsight.client.sslhacks.UnsafeClientConstructor;
-import org.apache.reef.runtime.yarn.YarnClasspathProvider;
import org.apache.reef.tang.formats.ConfigurationModule;
import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
import org.apache.reef.util.logging.LoggingSetup;
@@ -43,11 +43,11 @@ public final class UnsafeHDInsightRuntimeConfigurationStatic extends Configurati
public static final ConfigurationModule CONF = new UnsafeHDInsightRuntimeConfigurationStatic()
.bindImplementation(REEF.class, REEFImplementation.class)
- .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
.bindImplementation(RunningJob.class, RunningJobImpl.class)
.bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
.bindImplementation(JobSubmissionHandler.class, HDInsightJobSubmissionHandler.class)
.bindConstructor(CloseableHttpClient.class, UnsafeClientConstructor.class)
+ .bindImplementation(RuntimeClasspathProvider.class, HDInsightClasspathProvider.class)
.build();
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/AmContainerSpec.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/AmContainerSpec.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/AmContainerSpec.java
new file mode 100644
index 0000000..4b58ce2
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/AmContainerSpec.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents the specifications for an application master
+ * container. Used in job submission to the Resource Manager
+ * via the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class AmContainerSpec {
+
+ public static final String ACLS_VIEW_APP = "VIEW_APP";
+ public static final String ACLS_MODIFY_APP = "MODIFY_APP";
+
+ private static final String AM_CONTAINER_SPEC = "AmContainerSpec";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private Commands commands = new Commands();
+ private Map<String, List<StringEntry>> environment = new HashMap<>();
+ private Map<String, List<LocalResourcesEntry>> localResources = new HashMap<>();
+ private Map<String, List<StringEntry>> applicationAcls = new HashMap<>();
+ private Map<String, List<StringEntry>> serviceData = new HashMap<>();
+ private Credentials credentials;
+
+ public AmContainerSpec(){
+ this.localResources.put(Constants.ENTRY, new ArrayList<LocalResourcesEntry>());
+ this.environment.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ this.applicationAcls.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ this.serviceData.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+
+ public AmContainerSpec addEnvironment(final String key, final String value) {
+ if (!this.environment.containsKey(Constants.ENTRY)) {
+ this.environment.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+ this.environment.get(Constants.ENTRY).add(new StringEntry(key, value));
+ return this;
+ }
+
+ public AmContainerSpec addLocalResource(final String key, final LocalResource localResource) {
+ if (!this.localResources.containsKey(Constants.ENTRY)) {
+ this.localResources.put(Constants.ENTRY, new ArrayList<LocalResourcesEntry>());
+ }
+ this.localResources.get(Constants.ENTRY).add(new LocalResourcesEntry(key, localResource));
+ return this;
+ }
+
+ public AmContainerSpec addApplicationAcl(final String key, final String value) {
+ if (!this.applicationAcls.containsKey(Constants.ENTRY)) {
+ this.applicationAcls.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+ this.applicationAcls.get(Constants.ENTRY).add(new StringEntry(key, value));
+ return this;
+ }
+
+ public AmContainerSpec setCommand(final String command) {
+ this.commands.setCommand(command);
+ return this;
+ }
+
+ public AmContainerSpec addServiceData(final String key, final String value) {
+ if (!this.serviceData.containsKey(Constants.ENTRY)) {
+ this.serviceData.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+ this.serviceData.get(Constants.ENTRY).add(new StringEntry(key, value));
+ return this;
+ }
+
+ @JsonProperty(Constants.CREDENTIALS)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Credentials getCredentials() {
+ return this.credentials;
+ }
+
+ public AmContainerSpec setCredentials(final Credentials credentials) {
+ this.credentials = credentials;
+ return this;
+ }
+
+ @JsonProperty(Constants.SERVICE_DATA)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Map<String, List<StringEntry>> getServiceData() {
+ return this.serviceData;
+ }
+
+ public AmContainerSpec setServiceData(final Map<String, List<StringEntry>> serviceData) {
+ this.serviceData = serviceData;
+ return this;
+ }
+
+ @JsonProperty(Constants.APPLICATION_ACLS)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Map<String, List<StringEntry>> getApplicationAcls() {
+ return this.applicationAcls;
+ }
+
+ public AmContainerSpec setApplicationAcls(final Map<String, List<StringEntry>> applicationAcls) {
+ this.applicationAcls = applicationAcls;
+ return this;
+ }
+
+ @JsonProperty(Constants.ENVIRONMENT)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Map<String, List<StringEntry>> getEnvironment() {
+ return this.environment;
+ }
+
+ public void setEnvironment(final Map<String, List<StringEntry>> environment) {
+ this.environment = environment;
+ }
+
+ @JsonProperty(Constants.COMMANDS)
+ public Commands getCommands() {
+ return this.commands;
+ }
+
+ public AmContainerSpec setCommands(final Commands commands) {
+ this.commands = commands;
+ return this;
+ }
+
+ @JsonProperty(Constants.LOCAL_RESOURCES)
+ public Map<String, List<LocalResourcesEntry>> getLocalResources() {
+ return this.localResources;
+ }
+
+ public AmContainerSpec setLocalResources(final Map<String, List<LocalResourcesEntry>> localResources) {
+ this.localResources = localResources;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return AM_CONTAINER_SPEC + objectString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
index d3ead6b..c9ecbc7 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationID.java
@@ -18,22 +18,37 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
/**
- * Represents the response to an application ID request.
+ * Represents the response to an application ID request to
+ * the YARN Resource Manager via the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
*/
+@JsonIgnoreProperties(ignoreUnknown = true)
public final class ApplicationID {
- private String id;
+ private static final String APPLICATION_ID = "applicationId";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private String applicationId;
private Resource resource;
- public String getId() {
- return id;
+ @JsonProperty(Constants.APPLICATION_ID)
+ public String getApplicationId() {
+ return applicationId;
}
- public void setId(final String id) {
- this.id = id;
+ public void setApplicationId(final String applicationId) {
+ this.applicationId = applicationId;
}
+ @JsonProperty(Constants.MAXIMUM_RESOURCE_CAPABILITY)
public Resource getResource() {
return resource;
}
@@ -44,9 +59,15 @@ public final class ApplicationID {
@Override
public String toString() {
- return "ApplicationID{" +
- "id='" + id + '\'' +
- ", resource=" + resource +
- '}';
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return APPLICATION_ID + objectString;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
index 8586e5e..1a14426 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationResponse.java
@@ -18,27 +18,51 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
-import java.util.List;
-import java.util.Map;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
/**
- * Created by marku_000 on 2014-06-30.
+ * A response object used in deserialization when querying
+ * the Resource Manager for an application via the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
*/
-public class ApplicationResponse {
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ApplicationResponse {
+
+ private static String APPLICATION_RESPONSE = "applicationResponse";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private Map<String, List<ApplicationState>> apps;
+ private ApplicationState app;
- public Map<String, List<ApplicationState>> getApps() {
- return apps;
+ @JsonProperty(Constants.APP)
+ public ApplicationState getApp() {
+ return this.app;
}
- public void setApps(Map<String, List<ApplicationState>> apps) {
- this.apps = apps;
+ public void setApp(final ApplicationState app) {
+ this.app = app;
}
- public List<ApplicationState> getApplicationStates() {
- return apps.get("app");
+ public ApplicationState getApplicationState() {
+ return app;
}
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+ return APPLICATION_RESPONSE + objectString;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
index 6825dd9..680dca3 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationState.java
@@ -18,16 +18,31 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
/**
- * Created by marku_000 on 2014-06-30.
+ * An object representing the state of an application,
+ * used to deserialize queries for an application/list of applications
+ * to the Resource Manager on HDInsight via the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
*/
-public class ApplicationState {
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ApplicationState {
+
+ private static String APPLICATION_STATE = "applicationState";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
private String progress;
private String queue;
private String trackingUI;
private String state;
private String amContainerLogs;
- private String applicationType;
private int runningContainers;
private int allocatedMB;
private long elapsedTime;
@@ -37,29 +52,34 @@ public class ApplicationState {
private String trackingUrl;
private int allocatedVCores;
private long finishedTime;
- private String applicationTags;
private String name;
- private long clusterId;
+ private String applicationType;
+ private String clusterId;
private String user;
private String diagnostics;
private long startedTime;
+ private long memorySeconds;
+ private long vCoreSeconds;
- public String getProgress() {
- return progress;
+ @JsonProperty(Constants.FINISHED_TIME)
+ public long getFinishedTime() {
+ return finishedTime;
}
- public void setProgress(String progress) {
- this.progress = progress;
+ public void setFinishedTime(long finishedTime) {
+ this.finishedTime = finishedTime;
}
- public String getQueue() {
- return queue;
+ @JsonProperty(Constants.AM_CONTAINER_LOGS)
+ public String getAmContainerLogs() {
+ return amContainerLogs;
}
- public void setQueue(String queue) {
- this.queue = queue;
+ public void setAmContainerLogs(String amContainerLogs) {
+ this.amContainerLogs = amContainerLogs;
}
+ @JsonProperty(Constants.TRACKING_UI)
public String getTrackingUI() {
return trackingUI;
}
@@ -68,6 +88,7 @@ public class ApplicationState {
this.trackingUI = trackingUI;
}
+ @JsonProperty(Constants.STATE)
public String getState() {
return state;
}
@@ -76,46 +97,43 @@ public class ApplicationState {
this.state = state;
}
- public String getAmContainerLogs() {
- return amContainerLogs;
- }
-
- public void setAmContainerLogs(String amContainerLogs) {
- this.amContainerLogs = amContainerLogs;
- }
-
- public String getApplicationType() {
- return applicationType;
+ @JsonProperty(Constants.USER)
+ public String getUser() {
+ return user;
}
- public void setApplicationType(String applicationType) {
- this.applicationType = applicationType;
+ public void setUser(String user) {
+ this.user = user;
}
- public int getRunningContainers() {
- return runningContainers;
+ @JsonProperty(Constants.ID)
+ public String getId() {
+ return id;
}
- public void setRunningContainers(int runningContainers) {
- this.runningContainers = runningContainers;
+ public void setId(String id) {
+ this.id = id;
}
- public int getAllocatedMB() {
- return allocatedMB;
+ @JsonProperty(Constants.CLUSTER_ID)
+ public String getClusterId() {
+ return clusterId;
}
- public void setAllocatedMB(int allocatedMB) {
- this.allocatedMB = allocatedMB;
+ public void setClusterId(String clusterId) {
+ this.clusterId = clusterId;
}
- public long getElapsedTime() {
- return elapsedTime;
+ @JsonProperty(Constants.FINAL_STATUS)
+ public String getFinalStatus() {
+ return finalStatus;
}
- public void setElapsedTime(long elapsedTime) {
- this.elapsedTime = elapsedTime;
+ public void setFinalStatus(String finalStatus) {
+ this.finalStatus = finalStatus;
}
+ @JsonProperty(Constants.AM_HOST_HTTP_ADDRESS)
public String getAmHostHttpAddress() {
return amHostHttpAddress;
}
@@ -124,91 +142,134 @@ public class ApplicationState {
this.amHostHttpAddress = amHostHttpAddress;
}
- public String getId() {
- return id;
+ @JsonProperty(Constants.PROGRESS)
+ public String getProgress() {
+ return progress;
}
- public void setId(String id) {
- this.id = id;
+ public void setProgress(String progress) {
+ this.progress = progress;
}
- public String getFinalStatus() {
- return finalStatus;
+ @JsonProperty(Constants.NAME)
+ public String getName() {
+ return name;
}
- public void setFinalStatus(String finalStatus) {
- this.finalStatus = finalStatus;
+ public void setName(String name) {
+ this.name = name;
}
- public String getTrackingUrl() {
- return trackingUrl;
+ @JsonProperty(Constants.RESPONSE_APPLICATION_TYPE)
+ public String getApplicationType() {
+ return applicationType;
}
- public void setTrackingUrl(String trackingUrl) {
- this.trackingUrl = trackingUrl;
+ public void setApplicationType(String applicationType) {
+ this.applicationType = applicationType;
}
- public int getAllocatedVCores() {
- return allocatedVCores;
+ @JsonProperty(Constants.STARTED_TIME)
+ public long getStartedTime() {
+ return startedTime;
}
- public void setAllocatedVCores(int allocatedVCores) {
- this.allocatedVCores = allocatedVCores;
+ public void setStartedTime(long startedTime) {
+ this.startedTime = startedTime;
}
- public long getFinishedTime() {
- return finishedTime;
+ @JsonProperty(Constants.ELAPSED_TIME)
+ public long getElapsedTime() {
+ return elapsedTime;
}
- public void setFinishedTime(long finishedTime) {
- this.finishedTime = finishedTime;
+ public void setElapsedTime(long elapsedTime) {
+ this.elapsedTime = elapsedTime;
}
- public String getApplicationTags() {
- return applicationTags;
+ @JsonProperty(Constants.DIAGNOSTICS)
+ public String getDiagnostics() {
+ return diagnostics;
}
- public void setApplicationTags(String applicationTags) {
- this.applicationTags = applicationTags;
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
}
- public String getName() {
- return name;
+ @JsonProperty(Constants.TRACKING_URL)
+ public String getTrackingUrl() {
+ return trackingUrl;
}
- public void setName(String name) {
- this.name = name;
+ public void setTrackingUrl(String trackingUrl) {
+ this.trackingUrl = trackingUrl;
}
- public long getClusterId() {
- return clusterId;
+ @JsonProperty(Constants.QUEUE)
+ public String getQueue() {
+ return queue;
}
- public void setClusterId(long clusterId) {
- this.clusterId = clusterId;
+ public void setQueue(String queue) {
+ this.queue = queue;
}
- public String getUser() {
- return user;
+ @JsonProperty(Constants.ALLOCATED_MB)
+ public int getAllocatedMB() {
+ return allocatedMB;
}
- public void setUser(String user) {
- this.user = user;
+ public void setAllocatedMB(int allocatedMB) {
+ this.allocatedMB = allocatedMB;
}
- public String getDiagnostics() {
- return diagnostics;
+ @JsonProperty(Constants.ALLOCATED_VCORES)
+ public int getAllocatedVCores() {
+ return allocatedVCores;
}
- public void setDiagnostics(String diagnostics) {
- this.diagnostics = diagnostics;
+ public void setAllocatedVCores(int allocatedVCores) {
+ this.allocatedVCores = allocatedVCores;
}
- public long getStartedTime() {
- return startedTime;
+ @JsonProperty(Constants.RUNNING_CONTAINERS)
+ public int getRunningContainers() {
+ return runningContainers;
}
- public void setStartedTime(long startedTime) {
- this.startedTime = startedTime;
+ public void setRunningContainers(int runningContainers) {
+ this.runningContainers = runningContainers;
+ }
+
+ @JsonProperty(Constants.MEMORY_SECONDS)
+ public long getMemorySeconds() {
+ return memorySeconds;
+ }
+
+ public void setMemorySeconds(long memorySeconds) {
+ this.memorySeconds = memorySeconds;
+ }
+
+ @JsonProperty(Constants.VCORE_SECONDS)
+ public long getVCoreSeconds() {
+ return vCoreSeconds;
+ }
+
+ public void setVCoreSeconds(long vCoreSeconds) {
+ this.vCoreSeconds = vCoreSeconds;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return APPLICATION_STATE + objectString;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
index 77455e4..c79f9bb 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ApplicationSubmission.java
@@ -18,37 +18,51 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
/**
- * Represents an ApplicationSubmission to the YARN REST API.
+ * Represents an Application Submission object to the YARN REST API.
+ * Contains all the information needed to submit an application to the
+ * Resource Manager.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
*/
public final class ApplicationSubmission {
public static final String DEFAULT_QUEUE = "default";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final String APPLICATION_SUBMISSION = "applicationSubmission";
private String queue = DEFAULT_QUEUE;
- public static final String DEFAULT_PRIORITY = "3";
- private String priority = DEFAULT_PRIORITY;
+ public static final int DEFAULT_PRIORITY = 3;
+ private int priority = DEFAULT_PRIORITY;
- public static final String DEFAULT_MAX_ATTEMPTS = "1";
- private String maxAppAttempts = DEFAULT_MAX_ATTEMPTS;
+ public static final int DEFAULT_MAX_APP_ATTEMPTS = 1;
+ private int maxAppAttempts = DEFAULT_MAX_APP_ATTEMPTS;
public static final String DEFAULT_APPLICATION_TYPE = "YARN";
private String applicationType = DEFAULT_APPLICATION_TYPE;
- public static final String DEFAULT_KEEP_CONTAINERS = "false";
- private String keepContainers = DEFAULT_KEEP_CONTAINERS;
+ public static final boolean DEFAULT_KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS = false;
+ private boolean keepContainers = DEFAULT_KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS;
- public static final String DEFAULT_IS_UNMANAGED_AM = "false";
- private String isUnmanagedAM = DEFAULT_IS_UNMANAGED_AM;
-
- public static final String DEFAULT_CANCEL_TOKENS_WHEN_COMPLETE = "true";
- private String cancelTokensWhenComplete = DEFAULT_CANCEL_TOKENS_WHEN_COMPLETE;
+ public static final boolean DEFAULT_UNMANAGED_AM = false;
+ private boolean isUnmanagedAM = DEFAULT_UNMANAGED_AM;
private String applicationId;
private String applicationName;
- private ContainerInfo containerInfo;
+ private AmContainerSpec amContainerSpec;
private Resource resource;
+ private List<String> applicationTags = new ArrayList<>();
+ @JsonProperty(Constants.APPLICATION_ID)
public String getApplicationId() {
return applicationId;
}
@@ -58,6 +72,7 @@ public final class ApplicationSubmission {
return this;
}
+ @JsonProperty(Constants.APPLICATION_NAME)
public String getApplicationName() {
return applicationName;
}
@@ -67,6 +82,7 @@ public final class ApplicationSubmission {
return this;
}
+ @JsonProperty(Constants.APPLICATION_TYPE)
public String getApplicationType() {
return applicationType;
}
@@ -76,60 +92,57 @@ public final class ApplicationSubmission {
return this;
}
- public String isCancelTokensWhenComplete() {
- return cancelTokensWhenComplete;
- }
-
- public ApplicationSubmission setCancelTokensWhenComplete(String cancelTokensWhenComplete) {
- this.cancelTokensWhenComplete = cancelTokensWhenComplete;
- return this;
- }
-
- public ContainerInfo getContainerInfo() {
- return containerInfo;
+ @JsonProperty(Constants.AM_CONTAINER_SPEC)
+ public AmContainerSpec getAmContainerSpec() {
+ return amContainerSpec;
}
- public ApplicationSubmission setContainerInfo(ContainerInfo containerInfo) {
- this.containerInfo = containerInfo;
+ public ApplicationSubmission setAmContainerSpec(AmContainerSpec amContainerSpec) {
+ this.amContainerSpec = amContainerSpec;
return this;
}
- public String isUnmanagedAM() {
+ @JsonProperty(Constants.UNMANAGED_AM)
+ public boolean isUnmanagedAM() {
return isUnmanagedAM;
}
- public ApplicationSubmission setUnmanagedAM(String isUnmanagedAM) {
+ public ApplicationSubmission setUnmanagedAM(boolean isUnmanagedAM) {
this.isUnmanagedAM = isUnmanagedAM;
return this;
}
- public String isKeepContainers() {
+ @JsonProperty(Constants.KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS)
+ public boolean isKeepContainers() {
return keepContainers;
}
- public ApplicationSubmission setKeepContainers(String keepContainers) {
+ public ApplicationSubmission setKeepContainers(boolean keepContainers) {
this.keepContainers = keepContainers;
return this;
}
- public String getMaxAppAttempts() {
+ @JsonProperty(Constants.MAX_APP_ATTEMPTS)
+ public int getMaxAppAttempts() {
return maxAppAttempts;
}
- public ApplicationSubmission setMaxAppAttempts(String maxAppAttempts) {
+ public ApplicationSubmission setMaxAppAttempts(int maxAppAttempts) {
this.maxAppAttempts = maxAppAttempts;
return this;
}
- public String getPriority() {
+ @JsonProperty(Constants.PRIORITY)
+ public int getPriority() {
return priority;
}
- public ApplicationSubmission setPriority(String priority) {
+ public ApplicationSubmission setPriority(int priority) {
this.priority = priority;
return this;
}
+ @JsonProperty(Constants.QUEUE)
public String getQueue() {
return queue;
}
@@ -139,6 +152,7 @@ public final class ApplicationSubmission {
return this;
}
+ @JsonProperty(Constants.RESOURCE)
public Resource getResource() {
return resource;
}
@@ -148,20 +162,33 @@ public final class ApplicationSubmission {
return this;
}
+ public ApplicationSubmission addApplicationTag(String tag) {
+ this.applicationTags.add(tag);
+ return this;
+ }
+
+ @JsonProperty(Constants.APPLICATION_TAGS)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_EMPTY)
+ public List<String> getApplicationTags() {
+ return this.applicationTags;
+ }
+
+ public ApplicationSubmission setApplicationTags(final List<String> applicationTags) {
+ this.applicationTags = applicationTags;
+ return this;
+ }
+
@Override
public String toString() {
- return "ApplicationSubmission{" +
- "queue='" + queue + '\'' +
- ", priority=" + priority +
- ", maxAppAttempts=" + maxAppAttempts +
- ", applicationType='" + applicationType + '\'' +
- ", keepContainers=" + keepContainers +
- ", applicationId='" + applicationId + '\'' +
- ", applicationName='" + applicationName + '\'' +
- ", containerInfo=" + containerInfo +
- ", isUnmanagedAM=" + isUnmanagedAM +
- ", cancelTokensWhenComplete=" + cancelTokensWhenComplete +
- ", resource=" + resource +
- '}';
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return APPLICATION_SUBMISSION + objectString;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Commands.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Commands.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Commands.java
new file mode 100644
index 0000000..8008b39
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Commands.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+/**
+ * Commands used to start an Application Master via the
+ * YARN Resource Manger REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class Commands {
+
+ public static final String DEFAULT_COMMAND = "";
+
+ private static final String COMMANDS = "commands";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private String command = DEFAULT_COMMAND;
+
+ @JsonProperty(Constants.COMMAND)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_EMPTY)
+ public String getCommand() {
+ return this.command;
+ }
+
+ public void setCommand(final String command) {
+ this.command = command;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return COMMANDS + objectString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Constants.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Constants.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Constants.java
new file mode 100644
index 0000000..895b1fc
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Constants.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+/**
+ * Constants used in serializing/deserializing REST calls to HDInsight's
+ * Resource Manager.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class Constants {
+ public static final String ID = "id";
+ public static final String MAXIMUM_RESOURCE_CAPABILITY = "maximum-resource-capability";
+ public static final String APPLICATION_ID = "application-id";
+ public static final String APPLICATION_NAME = "application-name";
+ public static final String UNMANAGED_AM = "unmanaged-AM";
+ public static final String MAX_APP_ATTEMPTS = "max-app-attempts";
+ public static final String APPLICATION_TYPE = "application-type";
+ public static final String AM_CONTAINER_SPEC = "am-container-spec";
+ public static final String KEEP_CONTAINERS_ACROSS_APPLICATION_ATTEMPTS = "keep-containers-across-application-attempts";
+ public static final String APPLICATION_TAGS = "application-tags";
+ public static final String QUEUE = "queue";
+ public static final String RESOURCE = "resource";
+ public static final String PRIORITY = "priority";
+ public static final String LOCAL_RESOURCES = "local-resources";
+ public static final String ENVIRONMENT = "environment";
+ public static final String COMMANDS = "commands";
+ public static final String COMMAND = "command";
+ public static final String ENTRY = "entry";
+ public static final String KEY = "key";
+ public static final String VALUE = "value";
+ public static final String APPLICATION_ACLS = "application-acls";
+ public static final String SERVICE_DATA = "service-data";
+ public static final String CREDENTIALS = "credentials";
+ public static final String SECRETS = "secrets";
+ public static final String TOKENS = "tokens";
+ public static final String TYPE = "type";
+ public static final String VISIBILITY = "visibility";
+ public static final String SIZE = "size";
+ public static final String TIMESTAMP = "timestamp";
+ public static final String MEMORY = "memory";
+ public static final String VCORES = "vCores";
+ public static final String APPS = "apps";
+ public static final String APP = "app";
+ public static final String FINISHED_TIME = "finishedTime";
+ public static final String AM_CONTAINER_LOGS = "amContainerLogs";
+ public static final String TRACKING_UI = "trackingUI";
+ public static final String RESPONSE_APPLICATION_TYPE = "applicationType";
+ public static final String STATE = "state";
+ public static final String USER = "user";
+ public static final String CLUSTER_ID = "clusterId";
+ public static final String FINAL_STATUS = "finalStatus";
+ public static final String AM_HOST_HTTP_ADDRESS = "amHostHttpAddress";
+ public static final String PROGRESS = "progress";
+ public static final String NAME = "name";
+ public static final String STARTED_TIME = "startedTime";
+ public static final String ELAPSED_TIME = "elapsedTime";
+ public static final String DIAGNOSTICS = "diagnostics";
+ public static final String TRACKING_URL = "trackingUrl";
+ public static final String ALLOCATED_MB = "allocatedMB";
+ public static final String ALLOCATED_VCORES = "allocatedVCores";
+ public static final String RUNNING_CONTAINERS = "runningContainers";
+ public static final String MEMORY_SECONDS = "memorySeconds";
+ public static final String VCORE_SECONDS = "vcoreSeconds";
+
+ private Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
deleted file mode 100644
index 3c779d5..0000000
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ContainerInfo.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.reef.runtime.hdinsight.client.yarnrest;
-
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Represents a ContainerInfo in the YARN REST APIs.
- */
-public final class ContainerInfo {
-
- public static final String DEFAULT_SERVICE_DATA = null;
- private String serviceData = DEFAULT_SERVICE_DATA;
-
- public static final String DEFAULT_TOKENS = "";
- private String tokens = DEFAULT_TOKENS;
-
- public static final String DEFAULT_ACLS = null;
- private String acls = DEFAULT_ACLS;
-
- private List<String> commands = new ArrayList<>();
- private Map<String, EnvironmentEntry> environment = new HashMap<>();
- private Map<String, LocalResourcesEntry> localResources = new HashMap<>();
-
- /**
- * Adds an environment variable.
- *
- * @param key the name of the variable
- * @param value the value it shall take
- * @return this
- */
- public ContainerInfo addEnvironment(final String key, final String value) {
- this.environment.put("entry", new EnvironmentEntry(key, value));
- return this;
- }
-
- /**
- * Adds a command to the command list to be executed
- *
- * @param command
- * @return this
- */
- public ContainerInfo addCommand(final String command) {
- this.commands.add(command);
- return this;
- }
-
- public ContainerInfo addFileResource(final String key, final FileResource fileResource) {
- this.localResources.put("entry", new LocalResourcesEntry(key, fileResource));
- return this;
- }
-
- public String getServiceData() {
- return this.serviceData;
- }
-
- public ContainerInfo setServiceData(final String serviceData) {
- this.serviceData = serviceData;
- return this;
- }
-
- public String getTokens() {
- return this.tokens;
- }
-
- public ContainerInfo setTokens(final String tokens) {
- this.tokens = tokens;
- return this;
- }
-
- public String getAcls() {
- return this.acls;
- }
-
- public ContainerInfo setAcls(final String acls) {
- this.acls = acls;
- return this;
- }
-
- public Map<String, EnvironmentEntry> getEnvironment() {
- return this.environment;
- }
-
- public void setEnvironment(final Map<String, EnvironmentEntry> environment) {
- this.environment = environment;
- }
-
- public List<String> getCommands() {
- return this.commands;
- }
-
- public ContainerInfo setCommands(final List<String> commands) {
- this.commands = commands;
- return this;
- }
-
- public Map<String, LocalResourcesEntry> getLocalResources() {
- return this.localResources;
- }
-
- public ContainerInfo setLocalResources(final Map<String, LocalResourcesEntry> localResources) {
- this.localResources = localResources;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Credentials.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Credentials.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Credentials.java
new file mode 100644
index 0000000..a009f6e
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Credentials.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents credentials for an application in the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public class Credentials {
+
+ private static final String CREDENTIALS = "credentials";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private Map<String, List<StringEntry>> tokens = new HashMap<>();
+ private Map<String, List<StringEntry>> secrets = new HashMap<>();
+
+ public Credentials() {
+ this.tokens.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ this.secrets.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+
+ public Credentials addSecret(String key, String value) {
+ if (!this.secrets.containsKey(Constants.ENTRY)) {
+ this.secrets.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+ this.secrets.get(Constants.ENTRY).add(new StringEntry(key, value));
+ return this;
+ }
+
+ public Credentials addToken(String key, String value) {
+ if (!this.tokens.containsKey(Constants.ENTRY)) {
+ this.tokens.put(Constants.ENTRY, new ArrayList<StringEntry>());
+ }
+ this.tokens.get(Constants.ENTRY).add(new StringEntry(key, value));
+ return this;
+ }
+
+ @JsonProperty(Constants.SECRETS)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Map<String, List<StringEntry>> getSecrets() {
+ return this.secrets;
+ }
+
+ public Credentials setSecrets(final Map<String, List<StringEntry>> secrets) {
+ this.secrets = secrets;
+ return this;
+ }
+
+ @JsonProperty(Constants.TOKENS)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_DEFAULT)
+ public Map<String, List<StringEntry>> getTokens() {
+ return this.tokens;
+ }
+
+ public Credentials setTokens(final Map<String, List<StringEntry>> tokens) {
+ this.tokens = tokens;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return CREDENTIALS + objectString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
deleted file mode 100644
index 8444d47..0000000
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/EnvironmentEntry.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.reef.runtime.hdinsight.client.yarnrest;
-
-/**
- * An Entry in the Environment field of an ApplicationSubmission
- */
-public final class EnvironmentEntry {
-
- private String key;
- private String value;
-
- public EnvironmentEntry(final String key, final String value) {
- this.key = key;
- this.value = value;
- }
-
- public String getKey() {
- return this.key;
- }
-
- public void setKey(final String key) {
- this.key = key;
- }
-
- public String getValue() {
- return this.value;
- }
-
- public void setValue(final String value) {
- this.value = value;
- }
-
- @Override
- public String toString() {
- return "EnvironmentEntry{" +
- "key='" + this.key + '\'' +
- ", value='" + this.value + '\'' +
- '}';
- }
-
- @Override
- public boolean equals(final Object o) {
-
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- final EnvironmentEntry that = (EnvironmentEntry) o;
-
- return (this.key == that.key || (this.key != null && this.key.equals(that.key)))
- && (this.value == that.value || (this.value != null && this.value.equals(that.value)));
- }
-
- @Override
- public int hashCode() {
- int result = this.key != null ? this.key.hashCode() : 0;
- result = 31 * result + (this.value != null ? this.value.hashCode() : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
deleted file mode 100644
index 519228c..0000000
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/FileResource.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.reef.runtime.hdinsight.client.yarnrest;
-
-public final class FileResource {
-
- public static final String TYPE_FILE = "FILE";
- public static final String TYPE_ARCHIVE = "ARCHIVE";
-
- public static final String VISIBILITY_APPLICATION = "APPLICATION";
-
- private String url;
- private String type;
- private String visibility;
- private String size;
- private String timestamp;
-
- public String getUrl() {
- return this.url;
- }
-
- public FileResource setUrl(final String url) {
- this.url = url;
- return this;
- }
-
- public String getType() {
- return this.type;
- }
-
- public FileResource setType(final String type) {
- this.type = type;
- return this;
- }
-
- public String getVisibility() {
- return this.visibility;
- }
-
- public FileResource setVisibility(final String visibility) {
- this.visibility = visibility;
- return this;
- }
-
- public String getSize() {
- return this.size;
- }
-
- public FileResource setSize(final String size) {
- this.size = size;
- return this;
- }
-
- public String getTimestamp() {
- return this.timestamp;
- }
-
- public FileResource setTimestamp(final String timestamp) {
- this.timestamp = timestamp;
- return this;
- }
-
- @Override
- public String toString() {
- return "FileResource{" +
- "url='" + url + '\'' +
- ", type='" + type + '\'' +
- ", visibility='" + visibility + '\'' +
- ", size=" + size +
- ", timestamp=" + timestamp +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
index 2bbad8a..84071c8 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/HDInsightInstance.java
@@ -19,7 +19,6 @@
package org.apache.reef.runtime.hdinsight.client.yarnrest;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.http.Header;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
@@ -29,6 +28,7 @@ import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
@@ -58,14 +58,13 @@ import java.util.logging.Logger;
public final class HDInsightInstance {
private static final Logger LOG = Logger.getLogger(HDInsightInstance.class.getName());
- private static final String APPLICATION_KILL_MESSAGE = "{\"app:{\"state\":\"KILLED\"}}";
+ private static final String APPLICATION_KILL_MESSAGE = "{\"state\":\"KILLED\"}";
private final ObjectMapper objectMapper = new ObjectMapper();
private final Header[] headers;
private final HttpClientContext httpClientContext;
private final String instanceUrl;
- private final String username;
private final CloseableHttpClient httpClient;
@Inject
@@ -75,7 +74,6 @@ public final class HDInsightInstance {
final CloseableHttpClient client) throws URISyntaxException, IOException {
this.httpClient = client;
this.instanceUrl = instanceUrl.endsWith("/") ? instanceUrl : instanceUrl + "/";
- this.username = username;
final String host = this.getHost();
this.headers = new Header[]{
new BasicHeader("Host", host)
@@ -90,7 +88,7 @@ public final class HDInsightInstance {
* @throws IOException
*/
public ApplicationID getApplicationID() throws IOException {
- final String url = "ws/v1/cluster/appids?user.name=" + this.username;
+ final String url = "ws/v1/cluster/apps/new-application";
final HttpPost post = preparePost(url);
try (final CloseableHttpResponse response = this.httpClient.execute(post, this.httpClientContext)) {
final String message = IOUtils.toString(response.getEntity().getContent());
@@ -106,9 +104,7 @@ public final class HDInsightInstance {
* @throws IOException
*/
public void submitApplication(final ApplicationSubmission applicationSubmission) throws IOException {
-
- final String applicationId = applicationSubmission.getApplicationId();
- final String url = "ws/v1/cluster/apps/" + applicationId + "?user.name=" + this.username;
+ final String url = "ws/v1/cluster/apps";
final HttpPost post = preparePost(url);
final StringWriter writer = new StringWriter();
@@ -132,8 +128,26 @@ public final class HDInsightInstance {
*
* @param applicationId
*/
- public void killApplication(final String applicationId) {
- throw new NotImplementedException();
+ public void killApplication(final String applicationId) throws IOException {
+ final String url = this.getApplicationURL(applicationId) + "/state";
+ final HttpPut put = preparePut(url);
+ put.setEntity(new StringEntity(APPLICATION_KILL_MESSAGE, ContentType.APPLICATION_JSON));
+ this.httpClient.execute(put, this.httpClientContext);
+ }
+
+ /**
+ * Gets the application state given a YARN application ID.
+ * @param applicationId
+ * @return Application state of the requested application.
+ */
+ public ApplicationState getApplication(final String applicationId) throws IOException {
+ final String url = this.getApplicationURL(applicationId);
+ final HttpGet get = prepareGet(url);
+ try (final CloseableHttpResponse response = this.httpClient.execute(get, this.httpClientContext)) {
+ final String message = IOUtils.toString(response.getEntity().getContent());
+ final ApplicationResponse result = this.objectMapper.readValue(message, ApplicationResponse.class);
+ return result.getApplicationState();
+ }
}
public List<ApplicationState> listApplications() throws IOException {
@@ -141,7 +155,7 @@ public final class HDInsightInstance {
final HttpGet get = prepareGet(url);
try (final CloseableHttpResponse response = this.httpClient.execute(get, this.httpClientContext)) {
final String message = IOUtils.toString(response.getEntity().getContent());
- final ApplicationResponse result = this.objectMapper.readValue(message, ApplicationResponse.class);
+ final ListApplicationResponse result = this.objectMapper.readValue(message, ListApplicationResponse.class);
return result.getApplicationStates();
}
}
@@ -187,12 +201,26 @@ public final class HDInsightInstance {
return httpPost;
}
+ /**
+ * Creates a HttpPut request with all the common headers.
+ * @param url
+ * @return
+ */
+ private HttpPut preparePut(final String url) {
+ final HttpPut httpPut = new HttpPut(this.instanceUrl + url);
+ for (final Header header : this.headers) {
+ httpPut.addHeader(header);
+ }
+ return httpPut;
+ }
private HttpClientContext getClientContext(final String hostname, final String username, final String password) throws IOException {
final HttpHost targetHost = new HttpHost(hostname, 443, "https");
final HttpClientContext result = HttpClientContext.create();
+
// Setup credentials provider
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
result.setCredentialsProvider(credentialsProvider);
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ListApplicationResponse.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ListApplicationResponse.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ListApplicationResponse.java
new file mode 100644
index 0000000..acfe975
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/ListApplicationResponse.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The data structure used to deserialize the REST response
+ * from a call to the Resource Manager to list applications.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public final class ListApplicationResponse {
+
+ private static final String LIST_APPLICATION_RESPONSE = "listApplicationResponse";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private Map<String, List<ApplicationState>> apps;
+
+ @JsonProperty(Constants.APPS)
+ public Map<String, List<ApplicationState>> getApps() {
+ return apps;
+ }
+
+ public void setApps(final Map<String, List<ApplicationState>> apps) {
+ this.apps = apps;
+ }
+
+ public List<ApplicationState> getApplicationStates() {
+ if (!this.apps.containsKey(Constants.APP)) {
+ return null;
+ }
+ return apps.get(Constants.APP);
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return LIST_APPLICATION_RESPONSE + objectString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResource.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResource.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResource.java
new file mode 100644
index 0000000..ba81cfe
--- /dev/null
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResource.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.runtime.hdinsight.client.yarnrest;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+/**
+ * Represents a the details of a local resource used
+ * in an HDInsight job submission.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
+public final class LocalResource {
+
+ public static final String TYPE_FILE = "FILE";
+ public static final String TYPE_ARCHIVE = "ARCHIVE";
+ public static final String TYPE_PATTERN = "PATTERN";
+ public static final String VISIBILITY_PUBLIC = "PUBLIC";
+ public static final String VISIBILITY_PRIVATE = "PRIVATE";
+ public static final String VISIBILITY_APPLICATION = "APPLICATION";
+ private static final String LOCAL_RESOURCE = "localResource";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private String resource;
+ private String type;
+ private String visibility;
+ private long size;
+ private long timestamp;
+
+ @JsonProperty(Constants.RESOURCE)
+ public String getResource() {
+ return this.resource;
+ }
+
+ public LocalResource setResource(final String resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @JsonProperty(Constants.TYPE)
+ public String getType() {
+ return this.type;
+ }
+
+ public LocalResource setType(final String type) {
+ this.type = type;
+ return this;
+ }
+
+ @JsonProperty(Constants.VISIBILITY)
+ public String getVisibility() {
+ return this.visibility;
+ }
+
+ public LocalResource setVisibility(final String visibility) {
+ this.visibility = visibility;
+ return this;
+ }
+
+ @JsonProperty(Constants.SIZE)
+ public long getSize() {
+ return this.size;
+ }
+
+ public LocalResource setSize(final long size) {
+ this.size = size;
+ return this;
+ }
+
+ @JsonProperty(Constants.TIMESTAMP)
+ public long getTimestamp() {
+ return this.timestamp;
+ }
+
+ public LocalResource setTimestamp(final long timestamp) {
+ this.timestamp = timestamp;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return LOCAL_RESOURCE + objectString;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
index f73c31e..004a16f 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/LocalResourcesEntry.java
@@ -18,16 +18,33 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+/**
+ * Represents a resource to be localized. The key represents
+ * the file name or folder name after localization, while the value
+ * provides the details of the localized resource.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
+ */
public final class LocalResourcesEntry {
+ private static final String LOCAL_RESOURCES_ENTRY = "localResourcesEntry";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
private String key;
- private FileResource value;
+ private LocalResource value;
- public LocalResourcesEntry(final String key, final FileResource value) {
+ public LocalResourcesEntry(final String key, final LocalResource value) {
this.key = key;
this.value = value;
}
+ @JsonProperty(Constants.KEY)
public String getKey() {
return this.key;
}
@@ -37,12 +54,27 @@ public final class LocalResourcesEntry {
return this;
}
- public FileResource getValue() {
+ @JsonProperty(Constants.VALUE)
+ public LocalResource getValue() {
return this.value;
}
- public LocalResourcesEntry setValue(final FileResource value) {
+ public LocalResourcesEntry setValue(final LocalResource value) {
this.value = value;
return this;
}
+
+ @Override
+ public String toString() {
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return LOCAL_RESOURCES_ENTRY + objectString;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/16f4e073/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
index 7e1b647..366bff2 100644
--- a/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
+++ b/lang/java/reef-runtime-hdinsight/src/main/java/org/apache/reef/runtime/hdinsight/client/yarnrest/Resource.java
@@ -18,37 +18,55 @@
*/
package org.apache.reef.runtime.hdinsight.client.yarnrest;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
/**
- * Represents the resoure field in the YARN REST API
+ * Represents the resoure field in the YARN REST API.
+ * For detailed information, please refer to
+ * https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html
*/
public final class Resource {
- private String memory;
- private String vCores;
+ private static final String RESOURCE = "resource";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private int memory;
+ private int vCores;
- public String getMemory() {
+ @JsonProperty(Constants.MEMORY)
+ public int getMemory() {
return this.memory;
}
- public Resource setMemory(final String memory) {
+ public Resource setMemory(final int memory) {
this.memory = memory;
return this;
}
- public String getvCores() {
+ @JsonProperty(Constants.VCORES)
+ public int getvCores() {
return this.vCores;
}
- public Resource setvCores(final String vCores) {
+ public Resource setvCores(final int vCores) {
this.vCores = vCores;
return this;
}
@Override
public String toString() {
- return "Resource{" +
- "memory=" + this.memory +
- ", vCores=" + this.vCores +
- '}';
+ StringWriter writer = new StringWriter();
+ String objectString;
+ try {
+ OBJECT_MAPPER.writeValue(writer, this);
+ objectString = writer.toString();
+ } catch (IOException e) {
+ return null;
+ }
+
+ return RESOURCE + objectString;
}
}