You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2014/07/17 23:19:26 UTC
[5/6] KNOX-74: Support YARN REST API access via the Gateway
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/ExampleYarnApp.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/ExampleYarnApp.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/ExampleYarnApp.groovy
new file mode 100644
index 0000000..a12a283
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/ExampleYarnApp.groovy
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.yarn
+
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sandbox"
+username = "guest"
+password = "guest-password"
+
+hadoop = Hadoop.login( gateway, username, password )
+appId = Yarn.newApp(hadoop).now().getAppId()
+
+submitReq = """
+{
+ "application-id": "$appId",
+ "application-name": "test",
+ "am-container-spec": {
+ "local-resources": {
+ "entry": {
+ "key": "AppMaster.jar",
+ "value": {
+ "resource": "hdfs://localhost:9000/user/guest/AppMaster.jar",
+ "type": "FILE",
+ "visibility": "APPLICATION",
+ "size": "41601",
+ "timestamp": "1405544667528"
+ }
+ }
+ },
+ "commands": {
+ "command": "{{JAVA_HOME}}/bin/java -Xmx10m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_memory 10 --container_vcores 1 --num_containers 1 --priority 0 1><LOG_DIR>/AppMaster.stdout 2><LOG_DIR>/AppMaster.stderr"
+ },
+ "environment": {
+ "entry": [
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTTIMESTAMP",
+ "value": "1405545208994"
+ },
+ {
+ "key": "CLASSPATH",
+ "value": "{{CLASSPATH}}<CPS>./*<CPS>{{HADOOP_CONF_DIR}}<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/*<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/lib/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/lib/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/lib/*<CPS>./log4j.properties"
+ },
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTLEN",
+ "value": "50"
+ },
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTLOCATION",
+ "value": "hdfs://localhost:9000/user/guest/shellCommands.sh"
+ }
+ ]
+ }
+ },
+ "unmanaged-AM": "false",
+ "max-app-attempts": "2",
+ "resource": {
+ "memory": "1024",
+ "vCores": "1"
+ },
+ "application-type": "YARN",
+ "keep-containers-across-application-attempts": "false"
+}
+"""
+
+Yarn.submitApp(hadoop).text(submitReq).now()
+
+println(Yarn.appState(hadoop).appId(appId).now().getState())
+
+println(Yarn.killApp(hadoop).appId(appId).now().string)
+
+
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/KillApp.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/KillApp.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/KillApp.java
new file mode 100644
index 0000000..d3e7926
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/KillApp.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.yarn;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.BasicResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+
+import java.util.concurrent.Callable;
+
+public class KillApp {
+
+ static class Request extends AbstractRequest<Response> {
+
+ private String appId;
+
+ Request(Hadoop session) {
+ super(session);
+ }
+
+ public Request appId(String appId) {
+ this.appId = appId;
+ return this;
+ }
+
+ @Override
+ protected Callable<Response> callable() {
+ return new Callable<Response>() {
+ @Override
+ public Response call() throws Exception {
+ URIBuilder uri = uri( Yarn.SERVICE_PATH, "/v1/cluster/apps/", appId, "/state" );
+ HttpPut request = new HttpPut( uri.build() );
+ request.setEntity(new StringEntity("{ \"state\":\"KILLED\" }", ContentType.APPLICATION_JSON));
+ return new Response( execute( request ) );
+ }
+ };
+ }
+ }
+
+ static class Response extends BasicResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/NewApp.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/NewApp.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/NewApp.java
new file mode 100644
index 0000000..71bf6c5
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/NewApp.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.yarn;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.BasicResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+public class NewApp {
+
+ static class Request extends AbstractRequest<Response> {
+
+ protected Request(Hadoop session) {
+ super(session);
+ }
+
+ @Override
+ protected Callable<Response> callable() {
+ return new Callable<Response>() {
+ @Override
+ public Response call() throws Exception {
+ URIBuilder uri = uri( Yarn.SERVICE_PATH, "/v1/cluster/apps/new-application");
+ HttpPost request = new HttpPost( uri.build() );
+ return new Response( execute( request ) );
+ }
+ };
+ }
+ }
+
+ static class Response extends BasicResponse {
+
+ public Response(HttpResponse response) {
+ super(response);
+ }
+
+ public String getAppId() throws IOException {
+ return JsonPath.read(getString(), "$.application-id");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/SubmitApp.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/SubmitApp.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/SubmitApp.java
new file mode 100644
index 0000000..a58469d
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/SubmitApp.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.yarn;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.BasicResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.entity.StringEntity;
+
+import java.io.File;
+import java.util.concurrent.Callable;
+
+public class SubmitApp {
+
+ static class Request extends AbstractRequest<Response> {
+
+ private String text;
+ private String file;
+
+ Request(Hadoop session) {
+ super(session);
+ }
+
+ public Request text(String text) {
+ this.text = text;
+ return this;
+ }
+
+ public Request file(String file) {
+ this.file = file;
+ return this;
+ }
+
+ protected Callable<Response> callable() {
+ return new Callable<Response>() {
+ @Override
+ public Response call() throws Exception {
+ URIBuilder uri = uri(Yarn.SERVICE_PATH, "/v1/cluster/apps");
+ HttpPost request = new HttpPost(uri.build());
+ HttpEntity entity = null;
+ if (text != null) {
+ entity = new StringEntity(text, ContentType.APPLICATION_JSON);
+ } else if (file != null) {
+ entity = new FileEntity(new File(file), ContentType.APPLICATION_JSON);
+ }
+ request.setEntity(entity);
+ return new Response(execute(request));
+ }
+ };
+ }
+
+ }
+
+ static class Response extends BasicResponse {
+
+ Response(HttpResponse response) {
+ super(response);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/Yarn.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/Yarn.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/Yarn.java
new file mode 100644
index 0000000..3b0cfe9
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/yarn/Yarn.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.yarn;
+
+import org.apache.hadoop.gateway.shell.Hadoop;
+
+public class Yarn {
+
+ static String SERVICE_PATH = "/resourcemanager";
+
+ public static NewApp.Request newApp(Hadoop session) {
+ return new NewApp.Request(session);
+ }
+
+ public static SubmitApp.Request submitApp(Hadoop session) {
+ return new SubmitApp.Request(session);
+ }
+
+ public static KillApp.Request killApp(Hadoop session) {
+ return new KillApp.Request(session);
+ }
+
+ public static AppState.Request appState(Hadoop session) {
+ return new AppState.Request(session);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
index dd6dfe0..0eddaa0 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
@@ -20,13 +20,16 @@ package org.apache.hadoop.gateway;
import com.jayway.restassured.http.ContentType;
import com.jayway.restassured.response.Cookie;
import com.jayway.restassured.response.Response;
+import com.jayway.restassured.specification.ResponseSpecification;
import com.mycila.xmltool.XMLDoc;
import com.mycila.xmltool.XMLTag;
+
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.test.TestUtils;
import org.apache.hadoop.test.category.FunctionalTests;
import org.apache.hadoop.test.category.MediumTests;
import org.apache.hadoop.test.log.NoOpLogger;
+import org.apache.hadoop.test.mock.MockRequestMatcher;
import org.apache.http.HttpStatus;
import org.apache.velocity.Template;
import org.apache.velocity.VelocityContext;
@@ -34,7 +37,9 @@ import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.runtime.RuntimeConstants;
import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
import org.eclipse.jetty.util.log.Log;
+import org.hamcrest.Matcher;
import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -51,11 +56,16 @@ import java.io.IOException;
import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
+import java.net.URI;
import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
import static com.jayway.restassured.RestAssured.given;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -63,6 +73,7 @@ import static org.hamcrest.Matchers.greaterThan;
import static org.xmlmatchers.XmlMatchers.isEquivalentTo;
import static org.xmlmatchers.transform.XmlConverters.the;
import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs;
+import static org.hamcrest.text.IsEmptyString.isEmptyString;
@Category( { FunctionalTests.class, MediumTests.class } )
public class GatewayBasicFuncTest {
@@ -131,6 +142,7 @@ public class GatewayBasicFuncTest {
driver.setupService( "WEBHBASE", "http://" + TEST_HOST + ":60080", "/cluster/hbase", USE_MOCK_SERVICES );
driver.setupService( "NAMENODE", "hdfs://" + TEST_HOST + ":8020", null, USE_MOCK_SERVICES );
driver.setupService( "JOBTRACKER", "thrift://" + TEST_HOST + ":8021", null, USE_MOCK_SERVICES );
+ driver.setupService( "RESOURCEMANAGER", "http://" + TEST_HOST + ":8088/ws", "/cluster/resourcemanager", USE_MOCK_SERVICES );
driver.setupGateway( config, "cluster", createTopology(), USE_GATEWAY );
}
@@ -215,7 +227,10 @@ public class GatewayBasicFuncTest {
.addTag( "url" ).addText( driver.getRealUrl( "HIVE" ) ).gotoParent()
.addTag( "service" )
.addTag( "role" ).addText( "WEBHBASE" )
- .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) )
+ .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "RESOURCEMANAGER" )
+ .addTag( "url" ).addText( driver.getRealUrl( "RESOURCEMANAGER" ) ).gotoParent()
.gotoRoot();
// System.out.println( "GATEWAY=" + xml.toString() );
return xml;
@@ -1658,7 +1673,7 @@ public class GatewayBasicFuncTest {
.statusCode( HttpStatus.SC_OK )
.when().post( driver.getUrl( "WEBHBASE" ) + multipleRowPath );
driver.assertComplete();
-
+
driver.getMock( "WEBHBASE" )
.expect()
.method( "POST" )
@@ -1990,6 +2005,708 @@ public class GatewayBasicFuncTest {
driver.assertComplete();
}
+ @Test
+ public void testYarnRmGetClusterInfo() throws Exception {
+ getYarnRmResource( "/v1/cluster/", ContentType.JSON, "yarn/cluster-info" );
+ getYarnRmResource( "/v1/cluster/", ContentType.XML, "yarn/cluster-info" );
+ getYarnRmResource( "/v1/cluster/info/", ContentType.JSON, "yarn/cluster-info" );
+ getYarnRmResource( "/v1/cluster/info/", ContentType.XML, "yarn/cluster-info" );
+ }
+
+ @Test
+ public void testYarnRmGetClusterMetrics() throws Exception {
+ getYarnRmResource( "/v1/cluster/metrics/", ContentType.JSON, "yarn/cluster-metrics" );
+ getYarnRmResource( "/v1/cluster/metrics/", ContentType.XML, "yarn/cluster-metrics" );
+ }
+
+ @Test
+ public void testYarnRnGetScheduler() throws Exception {
+ getYarnRmResource( "/v1/cluster/scheduler/", ContentType.JSON, "yarn/scheduler" );
+ getYarnRmResource( "/v1/cluster/scheduler/", ContentType.XML, "yarn/scheduler" );
+ }
+
+ @Test
+ public void getYarnRmAppstatistics() throws Exception {
+ getYarnRmResource( "/v1/cluster/appstatistics/", ContentType.JSON, "yarn/appstatistics" );
+ getYarnRmResource( "/v1/cluster/appstatistics/", ContentType.XML, "yarn/appstatistics" );
+ }
+
+ @Test
+ public void testYarnRmGetApplications() throws Exception {
+ getYarnRmApps( ContentType.XML, null );
+ getYarnRmApps( ContentType.JSON, null );
+
+ Map<String, String> params = new HashMap<String, String>();
+ params.put( "states", "FINISHED" );
+ params.put( "finalStatus", "SUCCEEDED" );
+ params.put( "user", "test" );
+ params.put( "queue", "queueName" );
+ params.put( "limit", "100" );
+ params.put( "startedTimeBegin", "1399903578539" );
+ params.put( "startedTimeEnd", "1399903578539" );
+ params.put( "finishedTimeBegin", "1399904819572" );
+ params.put( "finishedTimeEnd", "1399904819572" );
+ params.put( "applicationTypes", "MAPREDUCE" );
+ params.put( "applicationTags", "a" );
+
+ getYarnRmApps( ContentType.XML, params );
+ getYarnRmApps( ContentType.JSON, params );
+ }
+
+ private void getYarnRmApps( ContentType contentType, Map<String,String> params ) throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/apps/";
+ String resource = "/yarn/apps";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path;
+ String gatewayPathQuery = driver.isUseGateway() ? "" : "?user.name=" + username;
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+
+ MockRequestMatcher mockRequestMatcher = driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path ).queryParam( "user.name", username );
+
+ if ( params != null ) {
+ for (Entry<String, String> param : params.entrySet()) {
+ mockRequestMatcher.queryParam( param.getKey(), param.getValue() );
+ if (gatewayPathQuery.isEmpty()) {
+ gatewayPathQuery += "?";
+ } else {
+ gatewayPathQuery += "&";
+ }
+ gatewayPathQuery += param.getKey() + "=" + param.getValue();
+ }
+ }
+
+
+ mockRequestMatcher.respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resource ) )
+ .contentType( contentType.toString() );
+
+ given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType )
+ .content( "apps.app[0].trackingUrl", isEmptyString() )
+ .content( "apps.app[1].trackingUrl", startsWith( "http://" + gatewayAddress.getHostName() + ":" + gatewayAddress.getPort() + "/" ) )
+ .content( "apps.app[0].amContainerLogs", isEmptyString() )
+ .content( "apps.app[1].amContainerLogs", isEmptyString() )
+ .content( "apps.app[0].amHostHttpAddress", isEmptyString() )
+ .content( "apps.app[1].amHostHttpAddress", isEmptyString() )
+ .content( "apps.app[2].id", is( "application_1399541193872_0009" ) )
+ .when()
+ .get( gatewayPath + gatewayPathQuery );
+
+ driver.assertComplete();
+ }
+
+ @Test
+ public void testYarnApplicationLifecycle() throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/apps/new-application";
+ String resource = "yarn/new-application.json";
+
+ driver.getMock("RESOURCEMANAGER")
+ .expect()
+ .method("POST")
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json");
+ Response response = given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode(HttpStatus.SC_OK)
+ .contentType("application/json")
+ .when().post(driver.getUrl("RESOURCEMANAGER") + path + (driver.isUseGateway() ? "" : "?user.name=" + username));
+ assertThat(response.getBody().asString(), Matchers.containsString("application-id"));
+
+ path = "/v1/cluster/apps";
+ resource = "yarn/application-submit-request.json";
+
+ driver.getMock("RESOURCEMANAGER")
+ .expect()
+ .method("POST")
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json")
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .contentType("application/json");
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json")
+ .expect()
+ .statusCode(HttpStatus.SC_OK)
+ .contentType("application/json")
+ .when().post(driver.getUrl("RESOURCEMANAGER") + path + (driver.isUseGateway() ? "" : "?user.name=" + username));
+ driver.assertComplete();
+
+ path = "/v1/cluster/apps/application_1405356982244_0031/state";
+ resource = "yarn/application-killing.json";
+ driver.getMock("RESOURCEMANAGER")
+ .expect()
+ .method("PUT")
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json")
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json");
+ response = given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content(driver.getResourceBytes(resource))
+ .contentType("application/json")
+ .expect()
+ .statusCode(HttpStatus.SC_OK)
+ .contentType("application/json")
+ .when().put(driver.getUrl("RESOURCEMANAGER") + path + (driver.isUseGateway() ? "" : "?user.name=" + username));
+ assertThat(response.getBody().asString(), Matchers.is("{\"state\":\"KILLING\"}"));
+ }
+
+ @Test
+ public void testYarnRmApplication() throws Exception {
+ getYarnRmApp( ContentType.JSON, true );
+ getYarnRmApp( ContentType.XML, true );
+ getYarnRmApp( ContentType.JSON, false );
+ getYarnRmApp( ContentType.XML, false );
+ }
+
+ private void getYarnRmApp( ContentType contentType, boolean running ) throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/apps/application_1399541193872_0033/";
+ String resource;
+ if ( running ) {
+ resource = "/yarn/app_running";
+ } else {
+ resource = "/yarn/app_succeeded";
+ }
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path + (driver.isUseGateway() ? "" : "?user.name=" + username);
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+
+ VelocityEngine velocity = new VelocityEngine();
+ velocity.setProperty( RuntimeConstants.RUNTIME_LOG_LOGSYSTEM_CLASS, "org.apache.velocity.runtime.log.NullLogSystem" );
+ velocity.setProperty( RuntimeConstants.RESOURCE_LOADER, "classpath" );
+ velocity.setProperty( "classpath.resource.loader.class", ClasspathResourceLoader.class.getName() );
+ velocity.init();
+
+ VelocityContext context = new VelocityContext();
+ context.put( "proxy_address", driver.getRealUrl( "RESOURCEMANAGER" ) );
+
+ String name = TestUtils.getResourceName( this.getClass(), resource );
+ Template template = velocity.getTemplate( name );
+ StringWriter sw = new StringWriter();
+ template.merge( context, sw );
+ String request = sw.toString();
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path ).queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( request.getBytes() )
+ .contentType( contentType.toString() );
+
+ ResponseSpecification response = given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType );
+ if ( running ) {
+ response.content( "app.trackingUrl", startsWith( "http://" + gatewayAddress.getHostName() + ":" + gatewayAddress.getPort() + "/" ) );
+ } else {
+ response.content( "app.trackingUrl", isEmptyString() );
+ }
+
+ response.content( "app.amContainerLogs", isEmptyString() )
+ .content( "app.amHostHttpAddress", isEmptyString() )
+ .when()
+ .get( gatewayPath );
+
+ driver.assertComplete();
+ }
+
+ private void getYarnRmResource( String path, ContentType contentType, String resource )
+ throws Exception {
+
+ String username = "hdfs";
+ String password = "hdfs-password";
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path ).queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resource ) )
+ .contentType( contentType.toString() );
+
+ Response response = given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType )
+ .when()
+ .get(
+ driver.getUrl( "RESOURCEMANAGER" ) + path
+ + (driver.isUseGateway() ? "" : "?user.name=" + username) );
+
+ switch( contentType ) {
+ case JSON:
+ MatcherAssert.assertThat( response.getBody().asString(),
+ sameJSONAs( driver.getResourceString( resource, UTF8 ) ) );
+ break;
+ case XML:
+ MatcherAssert
+ .assertThat( the( response.getBody().asString() ),
+ isEquivalentTo( the( driver.getResourceString( resource, UTF8 ) ) ) );
+ break;
+ default:
+ break;
+ }
+ driver.assertComplete();
+ }
+
+ @Test
+ public void testYarnRmAppattempts() throws Exception {
+ getYarnRmAppattempts( ContentType.JSON );
+ getYarnRmAppattempts( ContentType.XML );
+ }
+
+ private void getYarnRmAppattempts( ContentType contentType ) throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/apps/application_1399541193872_0018/appattempts/";
+ String resource = "/yarn/appattempts";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path + (driver.isUseGateway() ? "" : "?user.name=" + username);
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path ).queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resource ) )
+ .contentType( contentType.toString() );
+
+ given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType )
+ .content( "appAttempts.appAttempt[0].nodeHttpAddress", isEmptyString() )
+ .content( "appAttempts.appAttempt[0].nodeId", not( containsString( "localhost:50060" ) ) )
+ .content( "appAttempts.appAttempt[0].logsLink", isEmptyString() )
+ .when()
+ .get( gatewayPath );
+
+ driver.assertComplete();
+ }
+
+ @Test
+ public void testYarnRmNodes() throws Exception {
+ getYarnRmNodes( ContentType.JSON, null );
+ getYarnRmNodes( ContentType.XML, null );
+
+ Map<String, String> params = new HashMap<String, String>();
+ params.put( "state", "new,running" );
+ params.put( "healthy", "true" );
+
+ getYarnRmNodes( ContentType.JSON, params );
+ getYarnRmNodes( ContentType.XML, params );
+ }
+
+ private void getYarnRmNodes( ContentType contentType, Map<String, String> params ) throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/nodes/";
+ String nodesResource = "/yarn/nodes";
+ String nodeResource = "/yarn/node";
+ String nodeId = "localhost:45454";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path;
+ String gatewayPathQuery = driver.isUseGateway() ? "" : "?user.name=" + username;
+
+
+ MockRequestMatcher mockRequestMatcher = driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path ).queryParam( "user.name", username );
+
+ if ( params != null ) {
+ for (Entry<String, String> param : params.entrySet()) {
+ mockRequestMatcher.queryParam( param.getKey(), param.getValue() );
+ if (gatewayPathQuery.isEmpty()) {
+ gatewayPathQuery += "?";
+ } else {
+ gatewayPathQuery += "&";
+ }
+ gatewayPathQuery += param.getKey() + "=" + param.getValue();
+ }
+ }
+
+ mockRequestMatcher.respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( nodesResource + (contentType == ContentType.JSON ? ".json" : ".xml" ) ) )
+ .contentType( contentType.toString() );
+
+ String encryptedNodeId = given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType )
+ .content( "nodes.node[0].id", not( containsString( nodeId ) ) )
+ .content( "nodes.node[0].nodeHostName", isEmptyString() )
+ .content( "nodes.node[0].nodeHTTPAddress", isEmptyString() )
+ .when()
+ .get( gatewayPath + gatewayPathQuery ).getBody().path( "nodes.node[0].id" );
+
+ driver.assertComplete();
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path + nodeId ).queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( nodeResource + (contentType == ContentType.JSON ? ".json" : ".xml" ) ) )
+ .contentType( contentType.toString() );
+
+ given()
+// .log().all()
+ .auth()
+ .preemptive()
+ .basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( contentType )
+ .content( "node.id", not( containsString( nodeId ) ) )
+ .content( "node.nodeHostName", isEmptyString() )
+ .content( "node.nodeHTTPAddress", isEmptyString() )
+ .when()
+ .get( gatewayPath + encryptedNodeId );
+
+ driver.assertComplete();
+ }
+
+ @Test
+ public void testYarnRmProxy() throws Exception {
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String path = "/v1/cluster/apps/application_1399541193872_0033/";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path;
+
+ Map<String, Matcher<?>> matchers = new HashMap<String, Matcher<?>>();
+
+ VelocityEngine velocity = new VelocityEngine();
+ velocity.setProperty( RuntimeConstants.RUNTIME_LOG_LOGSYSTEM_CLASS, "org.apache.velocity.runtime.log.NullLogSystem" );
+ velocity.setProperty( RuntimeConstants.RESOURCE_LOADER, "classpath" );
+ velocity.setProperty( "classpath.resource.loader.class", ClasspathResourceLoader.class.getName() );
+ velocity.init();
+
+ VelocityContext context = new VelocityContext();
+ context.put( "proxy_address", driver.getRealUrl( "RESOURCEMANAGER" ) );
+
+ String name = TestUtils.getResourceName( this.getClass(), "yarn/app_running.json" );
+ Template template = velocity.getTemplate( name );
+ StringWriter sw = new StringWriter();
+ template.merge( context, sw );
+ String request = sw.toString();
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path )
+ .queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( request.getBytes() )
+ .contentType( ContentType.JSON.toString() );
+
+ String encryptedTrackingUrl = given()
+ // .log().all()
+ .auth().preemptive().basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+ // .log().all()
+ .statusCode( HttpStatus.SC_OK ).contentType( ContentType.JSON ).when()
+ .get( gatewayPath + ( driver.isUseGateway() ? "" : "?user.name=" + username ) ).getBody()
+ .path( "app.trackingUrl" );
+
+ String encryptedQuery = new URI( encryptedTrackingUrl ).getQuery();
+
+ driver.assertComplete();
+
+ // Test that root address of MapReduce Application Master REST API is not accessible through Knox
+ // For example, https://{gateway_host}:{gateway_port}/gateway/{cluster}/resourcemanager/proxy/{app_id}/?_={encrypted_application_proxy_location} should return Not Found response
+ // https://{gateway_host}:{gateway_port}/gateway/{cluster}/resourcemanager/proxy/{app_id}/ws/v1/mapreduce/?_={encrypted_application_proxy_location} returns OK
+ given()
+ // .log().all()
+ .auth().preemptive().basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" ).expect()
+ // .log().all()
+ .statusCode( HttpStatus.SC_NOT_FOUND ).when()
+ .get( encryptedTrackingUrl );
+
+ String resource = null;
+
+ path = "/proxy/application_1399541193872_0033/ws/v1/mapreduce/info";
+ resource = "yarn/proxy-mapreduce-info";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+ path = "/proxy/application_1399541193872_0033/ws/v1/mapreduce";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ path = "/proxy/application_1399541193872_0033/ws/v1/mapreduce/jobs";
+ resource = "yarn/proxy-mapreduce-jobs";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ path = "/proxy/application_1399541193872_0033/ws/v1/mapreduce/jobs/job_1399541193872_0035";
+ resource = "yarn/proxy-mapreduce-job";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ path = "/proxy/application_1399541193872_0033/ws/v1/mapreduce/jobs/job_1399541193872_0035/counters";
+ resource = "yarn/proxy-mapreduce-job-counters";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+
+
+// TODO: Need to understand what we should do with following properties
+// hadoop.proxyuser.HTTP.hosts
+// dfs.namenode.secondary.http-address
+// dfs.namenode.http-address
+// mapreduce.jobhistory.webapp.address
+// mapreduce.jobhistory.webapp.https.address
+// dfs.namenode.https-address
+// mapreduce.job.submithostname
+// yarn.resourcemanager.webapp.address
+// yarn.resourcemanager.hostname
+// mapreduce.jobhistory.address
+// yarn.resourcemanager.webapp.https.address
+// hadoop.proxyuser.oozie.hosts
+// hadoop.proxyuser.hive.hosts
+// dfs.namenode.secondary.https-address
+// hadoop.proxyuser.hcat.hosts
+// hadoop.proxyuser.HTTP.hosts
+// TODO: resolve java.util.regex.PatternSyntaxException: Unmatched closing ')' near index 17 m@\..*EXAMPLE\.COM)s
+ path = "/proxy/application_1399541193872_0035/ws/v1/mapreduce/jobs/job_1399541193872_0035/conf";
+ resource = "yarn/proxy-mapreduce-job-conf";
+// getYarnRmProxyJobConf( encryptedQuery, path, resource, ContentType.JSON );
+// getYarnRmProxyJobConf( encryptedQuery, path, resource, ContentType.XML );
+
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/jobattempts";
+ resource = "yarn/proxy-mapreduce-job-attempts";
+ matchers.clear();
+ matchers.put( "jobAttempts.jobAttempt[0].nodeHttpAddress", isEmptyString() );
+ matchers.put( "jobAttempts.jobAttempt[0].nodeId", not( containsString( "host.yarn.com:45454" ) ) );
+ matchers.put( "jobAttempts.jobAttempt[0].logsLink", isEmptyString() );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON, matchers );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML, matchers );
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks";
+ resource = "yarn/proxy-mapreduce-tasks";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks/task_1399541193872_0036_r_00";
+ resource = "yarn/proxy-mapreduce-task";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks/task_1399541193872_0036_r_00/counters";
+ resource = "yarn/proxy-mapreduce-task-counters";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks/task_1399541193872_0036_r_00/attempts";
+ resource = "yarn/proxy-mapreduce-task-attempts";
+ matchers.clear();
+ matchers.put( "taskAttempts.taskAttempt[0].nodeHttpAddress", isEmptyString() );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON, matchers );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML, matchers );
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks/task_1399541193872_0036_r_00/attempts/attempt_1399541193872_0036_r_000000_0";
+ resource = "yarn/proxy-mapreduce-task-attempt";
+ matchers.clear();
+ matchers.put( "taskAttempt.nodeHttpAddress", isEmptyString() );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON, matchers );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML, matchers );
+
+ path = "/proxy/application_1399541193872_0036/ws/v1/mapreduce/jobs/job_1399541193872_0036/tasks/task_1399541193872_0036_r_00/attempts/attempt_1399541193872_0036_r_000000_0/counters";
+ resource = "yarn/proxy-mapreduce-task-attempt-counters";
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.JSON );
+ getYarnRmProxyData( encryptedQuery, path, resource, ContentType.XML );
+
+ }
+
+ private void getYarnRmProxyData( String encryptedQuery, String path, String resource, ContentType contentType ) throws Exception {
+ getYarnRmProxyData( encryptedQuery, path, resource, contentType, null );
+ }
+
+ private void getYarnRmProxyData( String encryptedQuery, String path, String resource, ContentType contentType, Map<String, Matcher<?>> contentMatchers ) throws Exception {
+
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path + "?" + encryptedQuery + ( driver.isUseGateway() ? "" : "&user.name=" + username );
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path )
+ .queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resource ) )
+ .contentType( contentType.toString() );
+
+ ResponseSpecification responseSpecification = given()
+// .log().all()
+ .auth().preemptive().basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK ).contentType( contentType );
+
+ if ( contentMatchers != null ) {
+ for ( Entry<String, Matcher<?>> matcher : contentMatchers.entrySet() ) {
+ responseSpecification.content( matcher.getKey(), matcher.getValue() );
+ }
+ }
+
+ Response response = responseSpecification.when().get( gatewayPath );
+
+ if ( contentMatchers == null || contentMatchers.isEmpty() ) {
+ switch( contentType ) {
+ case JSON:
+ MatcherAssert.assertThat( response.getBody().asString(),
+ sameJSONAs( driver.getResourceString( resource, UTF8 ) ) );
+ break;
+ case XML:
+ MatcherAssert
+ .assertThat( the( response.getBody().asString() ),
+ isEquivalentTo( the( driver.getResourceString( resource, UTF8 ) ) ) );
+ break;
+ default:
+ break;
+ }
+ }
+
+ driver.assertComplete();
+ }
+
+ @SuppressWarnings("unused")
+ private void getYarnRmProxyJobConf( String encryptedQuery, String path, String resource, ContentType contentType ) throws Exception {
+
+ String username = "hdfs";
+ String password = "hdfs-password";
+ String gatewayPath = driver.getUrl( "RESOURCEMANAGER" ) + path + "?" + encryptedQuery + ( driver.isUseGateway() ? "" : "&user.name=" + username );
+
+ switch( contentType ) {
+ case JSON:
+ resource += ".json";
+ break;
+ case XML:
+ resource += ".xml";
+ break;
+ default:
+ break;
+ }
+
+ driver.getMock( "RESOURCEMANAGER" ).expect().method( "GET" )
+ .pathInfo( path )
+ .queryParam( "user.name", username ).respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resource ) )
+ .contentType( contentType.toString() );
+
+ Response response = given()
+// .log().all()
+ .auth().preemptive().basic( username, password )
+ .header( "X-XSRF-Header", "jksdhfkhdsf" )
+ .expect()
+// .log().all()
+ .statusCode( HttpStatus.SC_OK ).contentType( contentType ).when()
+ .get( gatewayPath );
+
+ assertThat( response.body().asString(), not( containsString( "host.yarn.com" ) ) );
+
+ driver.assertComplete();
+ }
+
private File findFile( File dir, String pattern ) {
File file = null;
FileFilter filter = new WildcardFileFilter( pattern );
@@ -2012,6 +2729,4 @@ public class GatewayBasicFuncTest {
}
return file.toURI().toString();
}
-
-
}
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.json
new file mode 100644
index 0000000..441dbc8
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.json
@@ -0,0 +1,25 @@
+{
+ "app":{
+ "id":"application_1399541193872_0033",
+ "user":"hdfs",
+ "name":"Sleep job",
+ "queue":"default",
+ "state":"RUNNING",
+ "finalStatus":"UNDEFINED",
+ "progress":89.94973,
+ "trackingUI":"ApplicationMaster",
+ "trackingUrl":"$proxy_address/proxy/application_1399541193872_0033/",
+ "diagnostics":"",
+ "clusterId":1399541193872,
+ "applicationType":"MAPREDUCE",
+ "applicationTags":"",
+ "startedTime":1401184778896,
+ "finishedTime":0,
+ "elapsedTime":139545,
+ "amContainerLogs":"http://localhost:50060/node/containerlogs/container_1399541193872_0033_01_000001/hdfs",
+ "amHostHttpAddress":"localhost:50060",
+ "allocatedMB":4608,
+ "allocatedVCores":2,
+ "runningContainers":2
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.xml
new file mode 100644
index 0000000..424f9df
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_running.xml
@@ -0,0 +1,40 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<app>
+ <id>application_1399541193872_0033</id>
+ <user>hdfs</user>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <state>RUNNING</state>
+ <finalStatus>UNDEFINED</finalStatus>
+ <progress>89.94973</progress>
+ <trackingUI>ApplicationMaster</trackingUI>
+ <trackingUrl>$proxy_address/proxy/application_1399541193872_0033/</trackingUrl>
+ <diagnostics></diagnostics>
+ <clusterId>1399541193872</clusterId>
+ <applicationType>MAPREDUCE</applicationType>
+ <applicationTags></applicationTags>
+ <startedTime>1401184778896</startedTime>
+ <finishedTime>0</finishedTime>
+ <elapsedTime>438390</elapsedTime>
+ <amContainerLogs>http://localhost:50060/node/containerlogs/container_1399541193872_0033_01_000001/hdfs</amContainerLogs>
+ <amHostHttpAddress>localhost:50060</amHostHttpAddress>
+ <allocatedMB>4608</allocatedMB>
+ <allocatedVCores>2</allocatedVCores>
+ <runningContainers>2</runningContainers>
+</app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.json
new file mode 100644
index 0000000..bb72949
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.json
@@ -0,0 +1,25 @@
+{
+ "app":{
+ "id":"application_1399541193872_0018",
+ "user":"hdfs",
+ "name":"Sleep job",
+ "queue":"default",
+ "state":"FINISHED",
+ "finalStatus":"SUCCEEDED",
+ "progress":100.0,
+ "trackingUI":"History",
+ "trackingUrl":"http://localhost:8088/proxy/application_1399541193872_0018/jobhistory/job/job_1399541193872_0018",
+ "diagnostics":"",
+ "clusterId":1399541193872,
+ "applicationType":"MAPREDUCE",
+ "applicationTags":"",
+ "startedTime":1399903578539,
+ "finishedTime":1399904819572,
+ "elapsedTime":1241033,
+ "amContainerLogs":"http://localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs",
+ "amHostHttpAddress":"localhost:50060",
+ "allocatedMB":0,
+ "allocatedVCores":0,
+ "runningContainers":0
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.xml
new file mode 100644
index 0000000..076ff95
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/app_succeeded.xml
@@ -0,0 +1,40 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<app>
+ <id>application_1399541193872_0018</id>
+ <user>hdfs</user>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <state>FINISHED</state>
+ <finalStatus>SUCCEEDED</finalStatus>
+ <progress>100.0</progress>
+ <trackingUI>History</trackingUI>
+ <trackingUrl>http://localhost:8088/proxy/application_1399541193872_0018/jobhistory/job/job_1399541193872_0018</trackingUrl>
+ <diagnostics></diagnostics>
+ <clusterId>1399541193872</clusterId>
+ <applicationType>MAPREDUCE</applicationType>
+ <applicationTags></applicationTags>
+ <startedTime>1399903578539</startedTime>
+ <finishedTime>1399904819572</finishedTime>
+ <elapsedTime>1241033</elapsedTime>
+ <amContainerLogs>http://localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs</amContainerLogs>
+ <amHostHttpAddress>localhost:50060</amHostHttpAddress>
+ <allocatedMB>0</allocatedMB>
+ <allocatedVCores>0</allocatedVCores>
+ <runningContainers>0</runningContainers>
+</app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.json
new file mode 100644
index 0000000..ef21483
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.json
@@ -0,0 +1,14 @@
+{
+ "appAttempts":{
+ "appAttempt":[
+ {
+ "id":1,
+ "startTime":1399903578541,
+ "containerId":"container_1399541193872_0018_01_000001",
+ "nodeHttpAddress":"localhost:50060",
+ "nodeId":"localhost:45454",
+ "logsLink":"//localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs"
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.xml
new file mode 100644
index 0000000..57a6378
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appattempts.xml
@@ -0,0 +1,27 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<appAttempts>
+ <appAttempt>
+ <id>1</id>
+ <startTime>1399903578541</startTime>
+ <containerId>container_1399541193872_0018_01_000001</containerId>
+ <nodeHttpAddress>localhost:50060</nodeHttpAddress>
+ <nodeId>localhost:45454</nodeId>
+ <logsLink>//localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs</logsLink>
+ </appAttempt>
+</appAttempts>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-killing.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-killing.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-killing.json
new file mode 100644
index 0000000..dda71d9
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-killing.json
@@ -0,0 +1 @@
+{"state":"KILLING"}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-submit-request.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-submit-request.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-submit-request.json
new file mode 100644
index 0000000..66cd45f
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/application-submit-request.json
@@ -0,0 +1,49 @@
+{
+ "application-id": "application_1405356982244_0031",
+ "application-name": "test",
+ "am-container-spec": {
+ "local-resources": {
+ "entry": {
+ "key": "AppMaster.jar",
+ "value": {
+ "resource": "hdfs://localhost:9000/user/hdfs/AppMaster.jar",
+ "type": "FILE",
+ "visibility": "APPLICATION",
+ "size": "41601",
+ "timestamp": "1405544667528"
+ }
+ }
+ },
+ "commands": {
+ "command": "{{JAVA_HOME}}/bin/java -Xmx10m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_memory 10 --container_vcores 1 --num_containers 1 --priority 0 1><LOG_DIR>/AppMaster.stdout 2><LOG_DIR>/AppMaster.stderr"
+ },
+ "environment": {
+ "entry": [
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTTIMESTAMP",
+ "value": "1405545208994"
+ },
+ {
+ "key": "CLASSPATH",
+ "value": "{{CLASSPATH}}<CPS>./*<CPS>{{HADOOP_CONF_DIR}}<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/*<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/lib/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/lib/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/lib/*<CPS>./log4j.properties"
+ },
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTLEN",
+ "value": "50"
+ },
+ {
+ "key": "DISTRIBUTEDSHELLSCRIPTLOCATION",
+ "value": "hdfs://localhost:9000/user/hdfs/shellCommands.sh"
+ }
+ ]
+ }
+ },
+ "unmanaged-AM": "false",
+ "max-app-attempts": "2",
+ "resource": {
+ "memory": "1024",
+ "vCores": "1"
+ },
+ "application-type": "YARN",
+ "keep-containers-across-application-attempts": "false"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.json
new file mode 100644
index 0000000..e6bd1f8
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.json
@@ -0,0 +1,72 @@
+{
+ "apps":{
+ "app":[
+ {
+ "id":"application_1399541193872_0018",
+ "user":"hdfs",
+ "name":"Sleep job",
+ "queue":"default",
+ "state":"FINISHED",
+ "finalStatus":"SUCCEEDED",
+ "progress":100.0,
+ "trackingUI":"History",
+ "trackingUrl":"http://localhost:8088/proxy/application_1399541193872_0018/jobhistory/job/job_1399541193872_0018",
+ "diagnostics":"",
+ "clusterId":1399541193872,
+ "applicationType":"MAPREDUCE",
+ "applicationTags":"",
+ "startedTime":1399903578539,
+ "finishedTime":1399904819572,
+ "elapsedTime":1241033,
+ "amContainerLogs":"http://localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs",
+ "amHostHttpAddress":"localhost:50060",
+ "allocatedMB":0,
+ "allocatedVCores":0,
+ "runningContainers":0
+ },
+ {
+ "id":"application_1399541193872_0031",
+ "user":"hdfs",
+ "name":"Sleep job",
+ "queue":"default",
+ "state":"RUNNING",
+ "finalStatus":"UNDEFINED",
+ "progress":89.94973,
+ "trackingUI":"ApplicationMaster",
+ "trackingUrl":"http://localhost:8088/proxy/application_1399541193872_0031/",
+ "diagnostics":"",
+ "clusterId":1399541193872,
+ "applicationType":"MAPREDUCE",
+ "applicationTags":"",
+ "startedTime":1400855314702,
+ "finishedTime":0,
+ "elapsedTime":471349,
+ "amContainerLogs":"http://localhost:50060/node/containerlogs/container_1399541193872_0031_01_000001/hdfs",
+ "amHostHttpAddress":"localhost:50060",
+ "allocatedMB":4608,
+ "allocatedVCores":2,
+ "runningContainers":2
+ },
+ {
+ "id":"application_1399541193872_0009",
+ "user":"hdfs",
+ "name":"Sleep job",
+ "queue":"a1",
+ "state":"FAILED",
+ "finalStatus":"FAILED",
+ "progress":0.0,
+ "trackingUI":"UNASSIGNED",
+ "diagnostics":"Application application_1399541193872_0009 submitted by user hdfs to unknown queue: a1",
+ "clusterId":1399541193872,
+ "applicationType":"MAPREDUCE",
+ "applicationTags":"",
+ "startedTime":1399543135010,
+ "finishedTime":1399543135013,
+ "elapsedTime":3,
+ "allocatedMB":0,
+ "allocatedVCores":0,
+ "runningContainers":0
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.xml
new file mode 100644
index 0000000..731c99c
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/apps.xml
@@ -0,0 +1,85 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<apps>
+ <app>
+ <id>application_1399541193872_0018</id>
+ <user>hdfs</user>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <state>FINISHED</state>
+ <finalStatus>SUCCEEDED</finalStatus>
+ <progress>100.0</progress>
+ <trackingUI>History</trackingUI>
+ <trackingUrl>http://localhost:8088/proxy/application_1399541193872_0018/jobhistory/job/job_1399541193872_0018</trackingUrl>
+ <diagnostics></diagnostics>
+ <clusterId>1399541193872</clusterId>
+ <applicationType>MAPREDUCE</applicationType>
+ <applicationTags></applicationTags>
+ <startedTime>1399903578539</startedTime>
+ <finishedTime>1399904819572</finishedTime>
+ <elapsedTime>1241033</elapsedTime>
+ <amContainerLogs>http://localhost:50060/node/containerlogs/container_1399541193872_0018_01_000001/hdfs</amContainerLogs>
+ <amHostHttpAddress>localhost:50060</amHostHttpAddress>
+ <allocatedMB>0</allocatedMB>
+ <allocatedVCores>0</allocatedVCores>
+ <runningContainers>0</runningContainers>
+ </app>
+ <app>
+ <id>application_1399541193872_0031</id>
+ <user>hdfs</user>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <state>RUNNING</state>
+ <finalStatus>UNDEFINED</finalStatus>
+ <progress>89.94973</progress>
+ <trackingUI>ApplicationMaster</trackingUI>
+ <trackingUrl>http://localhost:8088/proxy/application_1399541193872_0031/</trackingUrl>
+ <diagnostics></diagnostics>
+ <clusterId>1399541193872</clusterId>
+ <applicationType>MAPREDUCE</applicationType>
+ <applicationTags></applicationTags>
+ <startedTime>1400855314702</startedTime>
+ <finishedTime>0</finishedTime>
+ <elapsedTime>548138</elapsedTime>
+ <amContainerLogs>http://localhost:50060/node/containerlogs/container_1399541193872_0031_01_000001/hdfs</amContainerLogs>
+ <amHostHttpAddress>localhost:50060</amHostHttpAddress>
+ <allocatedMB>4608</allocatedMB>
+ <allocatedVCores>2</allocatedVCores>
+ <runningContainers>2</runningContainers>
+ </app>
+ <app>
+ <id>application_1399541193872_0009</id>
+ <user>hdfs</user>
+ <name>Sleep job</name>
+ <queue>a1</queue>
+ <state>FAILED</state>
+ <finalStatus>FAILED</finalStatus>
+ <progress>0.0</progress>
+ <trackingUI>UNASSIGNED</trackingUI>
+ <diagnostics>Application application_1399541193872_0009 submitted by user hdfs to unknown queue: a1</diagnostics>
+ <clusterId>1399541193872</clusterId>
+ <applicationType>MAPREDUCE</applicationType>
+ <applicationTags></applicationTags>
+ <startedTime>1399543135010</startedTime>
+ <finishedTime>1399543135013</finishedTime>
+ <elapsedTime>3</elapsedTime>
+ <allocatedMB>0</allocatedMB>
+ <allocatedVCores>0</allocatedVCores>
+ <runningContainers>0</runningContainers>
+ </app>
+</apps>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.json
new file mode 100644
index 0000000..894c7c6
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.json
@@ -0,0 +1,26 @@
+{
+ "appStatInfo":{
+ "statItem":[
+ {
+ "state":"FAILED",
+ "type":"mapreduce",
+ "count":2
+ },
+ {
+ "state":"FINISHED",
+ "type":"mapreduce",
+ "count":31
+ },
+ {
+ "state":"RUNNING",
+ "type":"mapreduce",
+ "count":1
+ },
+ {
+ "state":"ACCEPTED",
+ "type":"mapreduce",
+ "count":0
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.xml
new file mode 100644
index 0000000..a356c78
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/appstatistics.xml
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<appStatInfo>
+ <statItem>
+ <state>FAILED</state>
+ <type>mapreduce</type>
+ <count>2</count>
+ </statItem>
+ <statItem>
+ <state>FINISHED</state>
+ <type>mapreduce</type>
+ <count>31</count>
+ </statItem>
+ <statItem>
+ <state>RUNNING</state>
+ <type>mapreduce</type>
+ <count>1</count>
+ </statItem>
+ <statItem>
+ <state>ACCEPTED</state>
+ <type>mapreduce</type>
+ <count>0</count>
+ </statItem>
+</appStatInfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.json
new file mode 100644
index 0000000..fe49a72
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.json
@@ -0,0 +1,14 @@
+{
+ "clusterInfo":{
+ "id":1399541193872,
+ "startedOn":1399541193872,
+ "state":"STARTED",
+ "haState":"ACTIVE",
+ "resourceManagerVersion":"2.4.0.2.1.1.0-390",
+ "resourceManagerBuildVersion":"2.4.0.2.1.1.0-390 from 68ceccf06a4441273e81a5ec856d41fc7e11c792 by jenkins source checksum a06aa69de28a8ebc9ddba56f20d6d73d",
+ "resourceManagerVersionBuiltOn":"2014-04-23T16:39Z",
+ "hadoopVersion":"2.4.0.2.1.1.0-390",
+ "hadoopBuildVersion":"2.4.0.2.1.1.0-390 from 68ceccf06a4441273e81a5ec856d41fc7e11c792 by jenkins source checksum 9e788148daa5dd7934eb468e57e037b5",
+ "hadoopVersionBuiltOn":"2014-04-23T16:32Z"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.xml
new file mode 100644
index 0000000..dcd6a99
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-info.xml
@@ -0,0 +1,34 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<clusterInfo>
+ <id>1399541193872</id>
+ <startedOn>1399541193872</startedOn>
+ <state>STARTED</state>
+ <haState>ACTIVE</haState>
+ <resourceManagerVersion>2.4.0.2.1.1.0-390</resourceManagerVersion>
+ <resourceManagerBuildVersion>2.4.0.2.1.1.0-390 from
+ 68ceccf06a4441273e81a5ec856d41fc7e11c792 by jenkins source checksum
+ a06aa69de28a8ebc9ddba56f20d6d73d</resourceManagerBuildVersion>
+ <resourceManagerVersionBuiltOn>2014-04-23T16:39Z
+ </resourceManagerVersionBuiltOn>
+ <hadoopVersion>2.4.0.2.1.1.0-390</hadoopVersion>
+ <hadoopBuildVersion>2.4.0.2.1.1.0-390 from
+ 68ceccf06a4441273e81a5ec856d41fc7e11c792 by jenkins source checksum
+ 9e788148daa5dd7934eb468e57e037b5</hadoopBuildVersion>
+ <hadoopVersionBuiltOn>2014-04-23T16:32Z</hadoopVersionBuiltOn>
+</clusterInfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.json
new file mode 100644
index 0000000..35b7228
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.json
@@ -0,0 +1,23 @@
+{
+ "clusterMetrics":{
+ "appsSubmitted":28,
+ "appsCompleted":28,
+ "appsPending":0,
+ "appsRunning":0,
+ "appsFailed":0,
+ "appsKilled":0,
+ "reservedMB":0,
+ "availableMB":8192,
+ "allocatedMB":0,
+ "containersAllocated":0,
+ "containersReserved":0,
+ "containersPending":0,
+ "totalMB":8192,
+ "totalNodes":1,
+ "lostNodes":0,
+ "unhealthyNodes":0,
+ "decommissionedNodes":0,
+ "rebootedNodes":0,
+ "activeNodes":1
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.xml
new file mode 100644
index 0000000..e14e643
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/cluster-metrics.xml
@@ -0,0 +1,38 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<clusterMetrics>
+ <appsSubmitted>28</appsSubmitted>
+ <appsCompleted>28</appsCompleted>
+ <appsPending>0</appsPending>
+ <appsRunning>0</appsRunning>
+ <appsFailed>0</appsFailed>
+ <appsKilled>0</appsKilled>
+ <reservedMB>0</reservedMB>
+ <availableMB>8192</availableMB>
+ <allocatedMB>0</allocatedMB>
+ <containersAllocated>0</containersAllocated>
+ <containersReserved>0</containersReserved>
+ <containersPending>0</containersPending>
+ <totalMB>8192</totalMB>
+ <totalNodes>1</totalNodes>
+ <lostNodes>0</lostNodes>
+ <unhealthyNodes>0</unhealthyNodes>
+ <decommissionedNodes>0</decommissionedNodes>
+ <rebootedNodes>0</rebootedNodes>
+ <activeNodes>1</activeNodes>
+</clusterMetrics>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/new-application.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/new-application.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/new-application.json
new file mode 100644
index 0000000..a128f80
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/new-application.json
@@ -0,0 +1 @@
+{"application-id":"application_1405356982244_0031","maximum-resource-capability":{"memory":"8192","vCores":"32"}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.json
new file mode 100644
index 0000000..0d359fa
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.json
@@ -0,0 +1,15 @@
+{
+ "node":{
+ "rack":"/default-rack",
+ "state":"RUNNING",
+ "id":"localhost:45454",
+ "nodeHostName":"localhost",
+ "nodeHTTPAddress":"localhost:50060",
+ "lastHealthUpdate":1401197561733,
+ "version":"2.4.0.2.1.1.0-390",
+ "healthReport":"",
+ "numContainers":0,
+ "usedMemoryMB":0,
+ "availMemoryMB":8192
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.xml
new file mode 100644
index 0000000..2599ca3
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/node.xml
@@ -0,0 +1,30 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<node>
+ <rack>/default-rack</rack>
+ <state>RUNNING</state>
+ <id>localhost:45454</id>
+ <nodeHostName>localhost</nodeHostName>
+ <nodeHTTPAddress>localhost:50060</nodeHTTPAddress>
+ <lastHealthUpdate>1401197651742</lastHealthUpdate>
+ <version>2.4.0.2.1.1.0-390</version>
+ <healthReport></healthReport>
+ <numContainers>0</numContainers>
+ <usedMemoryMB>0</usedMemoryMB>
+ <availMemoryMB>8192</availMemoryMB>
+</node>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.json
new file mode 100644
index 0000000..0740757
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.json
@@ -0,0 +1,19 @@
+{
+ "nodes":{
+ "node":[
+ {
+ "rack":"/default-rack",
+ "state":"RUNNING",
+ "id":"localhost:45454",
+ "nodeHostName":"localhost",
+ "nodeHTTPAddress":"localhost:50060",
+ "lastHealthUpdate":1401194681758,
+ "version":"2.4.0.2.1.1.0-390",
+ "healthReport":"",
+ "numContainers":0,
+ "usedMemoryMB":0,
+ "availMemoryMB":8192
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.xml
new file mode 100644
index 0000000..619d995
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/nodes.xml
@@ -0,0 +1,32 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<nodes>
+ <node>
+ <rack>/default-rack</rack>
+ <state>RUNNING</state>
+ <id>localhost:45454</id>
+ <nodeHostName>hdp.example.com</nodeHostName>
+ <nodeHTTPAddress>localhost:50060</nodeHTTPAddress>
+ <lastHealthUpdate>1401194561733</lastHealthUpdate>
+ <version>2.4.0.2.1.1.0-390</version>
+ <healthReport></healthReport>
+ <numContainers>0</numContainers>
+ <usedMemoryMB>0</usedMemoryMB>
+ <availMemoryMB>8192</availMemoryMB>
+ </node>
+</nodes>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.json
new file mode 100644
index 0000000..b44a01b
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.json
@@ -0,0 +1,9 @@
+{
+ "info":{
+ "appId":"application_1399541193872_0035",
+ "name":"Sleep job",
+ "user":"hdfs",
+ "startedOn":1401199817106,
+ "elapsedTime":852931
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.xml
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.xml b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.xml
new file mode 100644
index 0000000..7ed4a49
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-info.xml
@@ -0,0 +1,24 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<info>
+ <appId>application_1399541193872_0035</appId>
+ <name>Sleep job</name>
+ <user>hdfs</user>
+ <startedOn>1401199817106</startedOn>
+ <elapsedTime>113773</elapsedTime>
+</info>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/a970502a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-job-attempts.json
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-job-attempts.json b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-job-attempts.json
new file mode 100644
index 0000000..af46d3e
--- /dev/null
+++ b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/yarn/proxy-mapreduce-job-attempts.json
@@ -0,0 +1,14 @@
+{
+ "jobAttempts":{
+ "jobAttempt":[
+ {
+ "nodeHttpAddress":"host.yarn.com:50060",
+ "nodeId":"host.yarn.com:45454",
+ "id":1,
+ "startTime":1401199817106,
+ "containerId":"container_1399541193872_0035_01_000001",
+ "logsLink":"http://host.yarn.com:50060/node/containerlogs/container_1399541193872_0035_01_000001/hdfs"
+ }
+ ]
+ }
+}
\ No newline at end of file