You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by mo...@apache.org on 2017/09/13 13:58:04 UTC
[07/11] knox git commit: KNOX-998 - Merging from master
http://git-wip-us.apache.org/repos/asf/knox/blob/582cc7e3/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
----------------------------------------------------------------------
diff --cc gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
index 1861148,0000000..0fbd6db
mode 100644,000000..100644
--- a/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/knox/gateway/GatewayBasicFuncTest.java
@@@ -1,4509 -1,0 +1,4515 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.knox.gateway;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.StringWriter;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import javax.ws.rs.core.MediaType;
+
- import com.jayway.restassured.http.ContentType;
- import com.jayway.restassured.path.json.JsonPath;
- import com.jayway.restassured.response.Cookie;
- import com.jayway.restassured.response.Header;
- import com.jayway.restassured.response.Response;
- import com.jayway.restassured.specification.ResponseSpecification;
++import io.restassured.RestAssured;
++import io.restassured.http.ContentType;
++import io.restassured.http.Cookie;
++import io.restassured.http.Header;
++import io.restassured.path.json.JsonPath;
++import io.restassured.response.Response;
++import io.restassured.specification.ResponseSpecification;
+import com.mycila.xmltool.XMLDoc;
+import com.mycila.xmltool.XMLTag;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.knox.gateway.util.KnoxCLI;
+import org.apache.hadoop.test.TestUtils;
+import org.apache.hadoop.test.category.MediumTests;
+import org.apache.hadoop.test.category.VerifyTest;
+import org.apache.hadoop.test.mock.MockRequestMatcher;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Matcher;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
- import static com.jayway.restassured.RestAssured.given;
++import static io.restassured.RestAssured.given;
+import static org.apache.hadoop.test.TestUtils.LOG_ENTER;
+import static org.apache.hadoop.test.TestUtils.LOG_EXIT;
+import static org.hamcrest.CoreMatchers.*;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.text.IsEmptyString.isEmptyString;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.xmlmatchers.XmlMatchers.isEquivalentTo;
+import static org.xmlmatchers.transform.XmlConverters.the;
+import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs;
+
+@Category( { VerifyTest.class, MediumTests.class } )
+public class GatewayBasicFuncTest {
+
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ // Uncomment to cause the test to hang after the gateway instance is setup.
+ // This will allow the gateway instance to be hit directly via some external client.
+// @Test
+// public void hang() throws IOException {
+// System.out.println( "Server on port " + driver.gateway.getAddresses()[0].getPort() );
+// System.out.println();
+// System.in.read();
+// }
+
+ private static Logger log = LoggerFactory.getLogger( GatewayBasicFuncTest.class );
+
+ private static GatewayTestDriver driver = new GatewayTestDriver();
+
+ // Controls the host name to which the gateway dispatch requests. This may be the name of a sandbox VM
+ // or an EC2 instance. Currently only a single host is supported.
+ private static final String TEST_HOST = "vm.local";
+
+ // Specifies if the test requests should go through the gateway or directly to the services.
+ // This is frequently used to verify the behavior of the test both with and without the gateway.
+ private static final boolean USE_GATEWAY = true;
+
+ // Specifies if the test requests should be sent to mock services or the real services.
+ // This is frequently used to verify the behavior of the test both with and without mock services.
+ private static final boolean USE_MOCK_SERVICES = true;
+
+ // Specifies if the GATEWAY_HOME created for the test should be deleted when the test suite is complete.
+ // This is frequently used during debugging to keep the GATEWAY_HOME around for inspection.
+ private static final boolean CLEANUP_TEST = true;
+
+// private static final boolean USE_GATEWAY = false;
+// private static final boolean USE_MOCK_SERVICES = false;
+// private static final boolean CLEANUP_TEST = false;
+
+ /**
+ * Creates a deployment of a gateway instance that all test methods will share. This method also creates a
+ * registry of sorts for all of the services that will be used by the test methods.
+ * The createTopology method is used to create the topology file that would normally be read from disk.
+ * The driver.setupGateway invocation is where the creation of GATEWAY_HOME occurs.
+ * @throws Exception Thrown if any failure occurs.
+ */
+ @BeforeClass
+ public static void setupSuite() throws Exception {
+ //Log.setLog( new NoOpLogger() );
+ LOG_ENTER();
+ GatewayTestConfig config = new GatewayTestConfig();
+ driver.setResourceBase(GatewayBasicFuncTest.class);
+ driver.setupLdap(0);
+ driver.setupService("WEBHDFS", "http://" + TEST_HOST + ":50070/webhdfs", "/cluster/webhdfs", USE_MOCK_SERVICES);
+ driver.setupService( "DATANODE", "http://" + TEST_HOST + ":50075/webhdfs", "/cluster/webhdfs/data", USE_MOCK_SERVICES );
+ driver.setupService( "WEBHCAT", "http://" + TEST_HOST + ":50111/templeton", "/cluster/templeton", USE_MOCK_SERVICES );
+ driver.setupService( "OOZIE", "http://" + TEST_HOST + ":11000/oozie", "/cluster/oozie", USE_MOCK_SERVICES );
+ driver.setupService( "HIVE", "http://" + TEST_HOST + ":10000", "/cluster/hive", USE_MOCK_SERVICES );
+ driver.setupService( "WEBHBASE", "http://" + TEST_HOST + ":60080", "/cluster/hbase", USE_MOCK_SERVICES );
+ driver.setupService( "NAMENODE", "hdfs://" + TEST_HOST + ":8020", null, USE_MOCK_SERVICES );
+ driver.setupService( "JOBTRACKER", "thrift://" + TEST_HOST + ":8021", null, USE_MOCK_SERVICES );
+ driver.setupService( "RESOURCEMANAGER", "http://" + TEST_HOST + ":8088/ws", "/cluster/resourcemanager", USE_MOCK_SERVICES );
+ driver.setupService( "FALCON", "http://" + TEST_HOST + ":15000", "/cluster/falcon", USE_MOCK_SERVICES );
+ driver.setupService( "STORM", "http://" + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES );
+ driver.setupService( "STORM-LOGVIEWER", "http://" + TEST_HOST + ":8477", "/cluster/storm", USE_MOCK_SERVICES );
+ driver.setupService( "SOLR", "http://" + TEST_HOST + ":8983", "/cluster/solr", USE_MOCK_SERVICES );
+ driver.setupService( "KAFKA", "http://" + TEST_HOST + ":8477", "/cluster/kafka", USE_MOCK_SERVICES );
+ driver.setupGateway( config, "cluster", createTopology(), USE_GATEWAY );
+ LOG_EXIT();
+ }
+
+ @AfterClass
+ public static void cleanupSuite() throws Exception {
+ LOG_ENTER();
+ if( CLEANUP_TEST ) {
+ driver.cleanup();
+ }
+ LOG_EXIT();
+ }
+
+ @After
+ public void cleanupTest() {
+ driver.reset();
+ }
+
+ /**
+ * Creates a topology that is deployed to the gateway instance for the test suite.
+ * Note that this topology is shared by all of the test methods in this suite.
+ * @return A populated XML structure for a topology file.
+ */
+ private static XMLTag createTopology() {
+ XMLTag xml = XMLDoc.newDocument( true )
+ .addRoot( "topology" )
+ .addTag( "gateway" )
+ .addTag( "provider" )
+ .addTag( "role" ).addText( "webappsec" )
+ .addTag("name").addText("WebAppSec")
+ .addTag("enabled").addText("true")
+ .addTag( "param" )
+ .addTag("name").addText("csrf.enabled")
+ .addTag("value").addText("true").gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("authentication")
+ .addTag("name").addText("ShiroProvider")
+ .addTag("enabled").addText("true")
+ .addTag( "param" )
+ .addTag("name").addText("main.ldapRealm")
+ .addTag("value").addText("org.apache.knox.gateway.shirorealm.KnoxLdapRealm").gotoParent()
+ .addTag( "param" )
+ .addTag( "name" ).addText( "main.ldapRealm.userDnTemplate" )
+ .addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent()
+ .addTag( "param" )
+ .addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" )
+ .addTag( "value" ).addText( driver.getLdapUrl() ).gotoParent()
+ .addTag( "param" )
+ .addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism" )
+ .addTag( "value" ).addText( "simple" ).gotoParent()
+ .addTag( "param" )
+ .addTag( "name" ).addText( "urls./**" )
+ .addTag( "value" ).addText( "authcBasic" ).gotoParent().gotoParent()
+ .addTag("provider")
+ .addTag("role").addText("identity-assertion")
+ .addTag("enabled").addText("true")
+ .addTag("name").addText("Default").gotoParent()
+ .addTag("provider")
+ .addTag( "role" ).addText( "authorization" )
+ .addTag( "enabled" ).addText( "true" )
+ .addTag("name").addText("AclsAuthz").gotoParent()
+ .addTag("param")
+ .addTag("name").addText( "webhdfs-acl" )
+ .addTag("value").addText( "hdfs;*;*" ).gotoParent()
+ .gotoRoot()
+ .addTag("service")
+ .addTag("role").addText("WEBHDFS")
+ .addTag("url").addText(driver.getRealUrl("WEBHDFS")).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "NAMENODE" )
+ .addTag( "url" ).addText( driver.getRealUrl( "NAMENODE" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "DATANODE" )
+ .addTag( "url" ).addText( driver.getRealUrl( "DATANODE" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "JOBTRACKER" )
+ .addTag( "url" ).addText( driver.getRealUrl( "JOBTRACKER" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "WEBHCAT" )
+ .addTag( "url" ).addText( driver.getRealUrl( "WEBHCAT" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "OOZIE" )
+ .addTag( "url" ).addText( driver.getRealUrl( "OOZIE" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "HIVE" )
+ .addTag( "url" ).addText( driver.getRealUrl( "HIVE" ) ).gotoParent()
+ .addTag( "service" )
+ .addTag( "role" ).addText( "WEBHBASE" )
+ .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("RESOURCEMANAGER")
+ .addTag("url").addText(driver.getRealUrl("RESOURCEMANAGER")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("FALCON")
+ .addTag("url").addText(driver.getRealUrl("FALCON")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("STORM")
+ .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("STORM-LOGVIEWER")
+ .addTag("url").addText(driver.getRealUrl("STORM-LOGVIEWER")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("SOLR")
+ .addTag("url").addText(driver.getRealUrl("SOLR")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("KAFKA")
+ .addTag("url").addText(driver.getRealUrl("KAFKA")).gotoParent()
+ .addTag("service")
+ .addTag("role").addText("SERVICE-TEST")
+ .gotoRoot();
+// System.out.println( "GATEWAY=" + xml.toString() );
+ return xml;
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicJsonUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testBasicJsonUseCase";
+ String username = "hdfs";
+ String password = "hdfs-password";
+ /* Create a directory.
+ curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+
+ The client receives a respond with a boolean JSON object:
+ HTTP/1.1 HttpStatus.SC_OK OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"boolean": true}
+ */
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir" )
+ .queryParam( "op", "MKDIRS" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "webhdfs-success.json" ) )
+ .contentType( "application/json" );
+ Cookie cookie = given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "MKDIRS" )
+ .expect()
+ //.log().all()
+ .statusCode( HttpStatus.SC_OK )
- .header( "Set-Cookie", containsString( "JSESSIONID" ) )
- .header( "Set-Cookie", containsString( "HttpOnly" ) )
+ .contentType( "application/json" )
+ .content( "boolean", is( true ) )
+ .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" ).getDetailedCookie( "JSESSIONID" );
+ assertThat( cookie.isSecured(), is( true ) );
++ assertThat( cookie.isHttpOnly(), is( true ) );
+ assertThat( cookie.getPath(), is( "/gateway/cluster" ) );
+ assertThat( cookie.getValue().length(), greaterThan( 16 ) );
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicOutboundHeaderUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testBasicOutboundHeaderUseCase";
+ String username = "hdfs";
+ String password = "hdfs-password";
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+ String gatewayHostName = gatewayAddress.getHostName();
+ String gatewayAddrName = InetAddress.getByName(gatewayHostName).getHostAddress();
+
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir/file" )
+ .header( "Host", driver.getRealAddr( "WEBHDFS" ) )
+ .queryParam( "op", "CREATE" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .header("Location", driver.getRealUrl("DATANODE") + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs");
+ Response response = given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "CREATE" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" );
+ String location = response.getHeader( "Location" );
+ //System.out.println( location );
+ log.debug( "Redirect location: " + response.getHeader( "Location" ) );
+ if( driver.isUseGateway() ) {
+ MatcherAssert.assertThat( location, anyOf(
+ startsWith( "http://" + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ),
+ startsWith( "http://" + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) );
+ MatcherAssert.assertThat( location, containsString( "?_=" ) );
+ }
+ MatcherAssert.assertThat(location, not(containsString("host=")));
+ MatcherAssert.assertThat(location, not(containsString("port=")));
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicOutboundEncodedHeaderUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testBasicOutboundHeaderUseCase";
+ String username = "hdfs";
+ String password = "hdfs-password";
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+ String gatewayHostName = gatewayAddress.getHostName();
+ String gatewayAddrName = InetAddress.getByName(gatewayHostName).getHostAddress();
+
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir/fileレポー" )
+ .header( "Host", driver.getRealAddr( "WEBHDFS" ) )
+ .queryParam( "op", "CREATE" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .header("Location", driver.getRealUrl("DATANODE") + "/v1" + root + "/dir/file%E3%83%AC%E3%83%9D%E3%83%BC?op=CREATE&user.name=hdfs");
+ Response response = given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "CREATE" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/fileレポー" );
+// .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file%E3%83%AC%E3%83%9D%E3%83%BC" );
+ String location = response.getHeader( "Location" );
+ //System.out.println( location );
+ log.debug( "Redirect location: " + response.getHeader( "Location" ) );
+ if( driver.isUseGateway() ) {
+ MatcherAssert.assertThat( location, containsString("/dir/file%E3%83%AC%E3%83%9D%E3%83%BC") );
+ }
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHdfsTildeUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testHdfsTildeUseCase";
+ String username = "hdfs";
+ String password = "hdfs-password";
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+
+ // Attempt to delete the test directory in case a previous run failed.
+ // Ignore any result.
+ // Cleanup anything that might have been leftover because the test failed previously.
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "DELETE" )
+ .from( "testHdfsTildeUseCase" )
+ .pathInfo( "/v1/user/hdfs" + root )
+ .queryParam( "op", "DELETE" )
+ .queryParam( "user.name", username )
+ .queryParam( "recursive", "true" )
+ .respond()
+ .status( HttpStatus.SC_OK );
- given()
- //.log().all()
- .auth().preemptive().basic( username, password )
- .header("X-XSRF-Header", "jksdhfkhdsf")
- .queryParam( "op", "DELETE" )
- .queryParam( "recursive", "true" )
- .expect()
- //.log().all()
- .statusCode( HttpStatus.SC_OK )
- .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1/~" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) );
- driver.assertComplete();
+
- driver.getMock( "WEBHDFS" )
- .expect()
- .method( "PUT" )
- .pathInfo( "/v1/user/hdfs/dir" )
- .queryParam( "op", "MKDIRS" )
- .queryParam( "user.name", username )
- .respond()
- .status( HttpStatus.SC_OK )
- .content( driver.getResourceBytes( "webhdfs-success.json" ) )
- .contentType("application/json");
- given()
- //.log().all()
- .auth().preemptive().basic( username, password )
- .header("X-XSRF-Header", "jksdhfkhdsf")
- .queryParam( "op", "MKDIRS" )
- .expect()
- //.log().all();
- .statusCode( HttpStatus.SC_OK )
- .contentType( "application/json" )
- .content( "boolean", is( true ) )
- .when().put( driver.getUrl( "WEBHDFS" ) + "/v1/~/dir" );
- driver.assertComplete();
++ try {
++ // Need to turn off URL encoding here or otherwise the tilde gets encoded and the rewrite rules fail
++ RestAssured.urlEncodingEnabled = false;
++ given()
++ //.log().all()
++ .auth().preemptive().basic( username, password )
++ .header("X-XSRF-Header", "jksdhfkhdsf")
++ .queryParam( "op", "DELETE" )
++ .queryParam( "recursive", "true" )
++ .expect()
++ //.log().all()
++ .statusCode( HttpStatus.SC_OK )
++ .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1/~" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) );
++ driver.assertComplete();
++
++ driver.getMock( "WEBHDFS" )
++ .expect()
++ .method( "PUT" )
++ .pathInfo( "/v1/user/hdfs/dir" )
++ .queryParam( "op", "MKDIRS" )
++ .queryParam( "user.name", username )
++ .respond()
++ .status( HttpStatus.SC_OK )
++ .content( driver.getResourceBytes( "webhdfs-success.json" ) )
++ .contentType("application/json");
++ given()
++ //.log().all()
++ .auth().preemptive().basic( username, password )
++ .header("X-XSRF-Header", "jksdhfkhdsf")
++ .queryParam( "op", "MKDIRS" )
++ .expect()
++ //.log().all();
++ .statusCode( HttpStatus.SC_OK )
++ .contentType( "application/json" )
++ .content( "boolean", is( true ) )
++ .when().put( driver.getUrl( "WEBHDFS" ) + "/v1/~/dir" );
++ driver.assertComplete();
++ } finally {
++ RestAssured.urlEncodingEnabled = true;
++ }
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicHdfsUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testBasicHdfsUseCase";
+ String username = "hdfs";
+ String password = "hdfs-password";
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+ String gatewayHostName = gatewayAddress.getHostName();
+ String gatewayAddrName = InetAddress.getByName( gatewayHostName ).getHostAddress();
+
+ // Attempt to delete the test directory in case a previous run failed.
+ // Ignore any result.
+ // Cleanup anything that might have been leftover because the test failed previously.
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "DELETE" )
+ .from( "testBasicHdfsUseCase-1" )
+ .pathInfo( "/v1" + root )
+ .queryParam( "op", "DELETE" )
+ .queryParam( "user.name", username )
+ .queryParam( "recursive", "true" )
+ .respond()
+ .status( HttpStatus.SC_OK );
+ given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "DELETE" )
+ .queryParam( "recursive", "true" )
+ .expect()
+ //.log().all()
+ .statusCode( HttpStatus.SC_OK )
+ .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1" + root + ( driver.isUseGateway() ? "" : "?user.name=" + username ) );
+ driver.assertComplete();
+
+ /* Create a directory.
+ curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
+
+ The client receives a respond with a boolean JSON object:
+ HTTP/1.1 HttpStatus.SC_OK OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"boolean": true}
+ */
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir" )
+ .queryParam( "op", "MKDIRS" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "webhdfs-success.json" ) )
+ .contentType( "application/json" );
+ given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "MKDIRS" )
+ .expect()
+ //.log().all();
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( "application/json" )
+ .content( "boolean", is( true ) )
+ .when().put( driver.getUrl( "WEBHDFS" ) + "/v1" + root + "/dir" );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/v1" + root )
+ .queryParam( "op", "LISTSTATUS" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "webhdfs-liststatus-test.json" ) )
+ .contentType( "application/json" );
+ given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "LISTSTATUS" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_OK )
+ .content( "FileStatuses.FileStatus[0].pathSuffix", is( "dir" ) )
+ .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );
+ driver.assertComplete();
+
+ //NEGATIVE: Test a bad password.
+ given()
+ //.log().all()
+ .auth().preemptive().basic( username, "invalid-password" )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "LISTSTATUS" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_UNAUTHORIZED )
+ .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );
+ driver.assertComplete();
+
+ //NEGATIVE: Test a bad user.
+ given()
+ //.log().all()
+ .auth().preemptive().basic( "hdfs-user", "hdfs-password" )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "LISTSTATUS" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_UNAUTHORIZED )
+ .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );
+ driver.assertComplete();
+
+ //NEGATIVE: Test a valid but unauthorized user.
+ given()
+ //.log().all()
+ .auth().preemptive().basic( "mapred-user", "mapred-password" )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "LISTSTATUS" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_UNAUTHORIZED )
+ .when().get( driver.getUrl( "WEBHDFS" ) + "/v1" + root );
+
+ /* Add a file.
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
+ [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
+ [&permission=<OCTAL>][&buffersize=<INT>]"
+
+ The expect is redirected to a datanode where the file data is to be written:
+ HTTP/1.1 307 TEMPORARY_REDIRECT
+ Location: http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=CREATE...
+ Content-Length: 0
+
+ Step 2: Submit another HTTP PUT expect using the URL in the Location header with the file data to be written.
+ curl -i -X PUT -T <LOCAL_FILE> "http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=CREATE..."
+
+ The client receives a HttpStatus.SC_CREATED Created respond with zero content length and the WebHDFS URI of the file in the Location header:
+ HTTP/1.1 HttpStatus.SC_CREATED Created
+ Location: webhdfs://<HOST>:<PORT>/<PATH>
+ Content-Length: 0
+ */
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir/file" )
+ .queryParam( "op", "CREATE" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .header( "Location", driver.getRealUrl( "DATANODE" ) + "/v1" + root + "/dir/file?op=CREATE&user.name=hdfs" );
+ driver.getMock( "DATANODE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( "/v1" + root + "/dir/file" )
+ .queryParam( "op", "CREATE" )
+ .queryParam( "user.name", username )
+ .contentType( "text/plain" )
+ .content( driver.getResourceBytes( "test.txt" ) )
+ //.content( driver.gerResourceBytes( "hadoop-examples.jar" ) )
+ .respond()
+ .status( HttpStatus.SC_CREATED )
+ .header( "Location", "webhdfs://" + driver.getRealAddr( "DATANODE" ) + "/v1" + root + "/dir/file" );
+ Response response = given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "CREATE" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .when().put( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" );
+ String location = response.getHeader( "Location" );
+ log.debug( "Redirect location: " + response.getHeader( "Location" ) );
+ if( driver.isUseGateway() ) {
+ MatcherAssert.assertThat( location, anyOf(
+ startsWith( "http://" + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ),
+ startsWith( "http://" + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) );
+ MatcherAssert.assertThat( location, containsString( "?_=" ) );
+ }
+ MatcherAssert.assertThat( location, not( containsString( "host=" ) ) );
+ MatcherAssert.assertThat( location, not( containsString( "port=" ) ) );
+ response = given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "test.txt" ) )
+ .contentType( "text/plain" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_CREATED )
+ .when().put( location );
+ location = response.getHeader( "Location" );
+ log.debug( "Created location: " + location );
+ if( driver.isUseGateway() ) {
+ MatcherAssert.assertThat( location, anyOf(
+ startsWith( "http://" + gatewayHostName + ":" + gatewayAddress.getPort() + "/" ),
+ startsWith( "http://" + gatewayAddrName + ":" + gatewayAddress.getPort() + "/" ) ) );
+ }
+ driver.assertComplete();
+
+ /* Get the file.
+ curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
+ [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
+
+ The expect is redirected to a datanode where the file data can be read:
+ HTTP/1.1 307 TEMPORARY_REDIRECT
+ Location: http://<DATANODE>:<PORT>/webhdfs/v1/<PATH>?op=OPEN...
+ Content-Length: 0
+
+ The client follows the redirect to the datanode and receives the file data:
+ HTTP/1.1 HttpStatus.SC_OK OK
+ Content-Type: application/octet-stream
+ Content-Length: 22
+
+ Hello, webhdfs user!
+ */
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/v1" + root + "/dir/file" )
+ .queryParam( "op", "OPEN" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_TEMPORARY_REDIRECT )
+ .header( "Location", driver.getRealUrl( "DATANODE" ) + "/v1" + root + "/dir/file?op=OPEN&user.name=hdfs" );
+ driver.getMock( "DATANODE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/v1" + root + "/dir/file" )
+ .queryParam( "op", "OPEN" )
+ .queryParam( "user.name", username )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .contentType( "text/plain" )
+ .content( driver.getResourceBytes( "test.txt" ) );
+ given()
+ //.log().all()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "OPEN" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_OK )
+ .content( is( "TEST" ) )
+ .when().get( driver.getUrl("WEBHDFS") + "/v1" + root + "/dir/file" );
+ driver.assertComplete();
+
+ /* Delete the directory.
+ curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
+ [&recursive=<true|false>]"
+
+ The client receives a respond with a boolean JSON object:
+ HTTP/1.1 HttpStatus.SC_OK OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"boolean": true}
+ */
+ // Mock the interaction with the namenode.
+ driver.getMock( "WEBHDFS" )
+ .expect()
+ .from( "testBasicHdfsUseCase-1" )
+ .method( "DELETE" )
+ .pathInfo( "/v1" + root )
+ .queryParam( "op", "DELETE" )
+ .queryParam( "user.name", username )
+ .queryParam( "recursive", "true" )
+ .respond()
+ .status( HttpStatus.SC_OK );
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "DELETE" )
+ .queryParam( "recursive", "true" )
+ .expect()
+ //.log().ifError()
+ .statusCode( HttpStatus.SC_OK )
+ .when().delete( driver.getUrl( "WEBHDFS" ) + "/v1" + root );
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ // User hdfs in groups hadoop, hdfs
+ // User mapred in groups hadoop, mapred
+ // User hcat in group hcat
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testPmHdfsM1UseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testPmHdfdM1UseCase";
+ String userA = "hdfs";
+ String passA = "hdfs-password";
+ String userB = "mapred";
+ String passB = "mapred-password";
+ String userC = "hcat";
+ String passC = "hcat-password";
+ String groupA = "hdfs";
+ String groupB = "mapred";
+ String groupAB = "hadoop";
+ String groupC = "hcat";
+
+ deleteFile( userA, passA, root, "true", 200 );
+
+ createDir( userA, passA, groupA, root + "/dirA700", "700", 200, 200 );
+ createDir( userA, passA, groupA, root + "/dirA770", "770", 200, 200 );
+ createDir( userA, passA, groupA, root + "/dirA707", "707", 200, 200 );
+ createDir( userA, passA, groupA, root + "/dirA777", "777", 200, 200 );
+ createDir( userA, passA, groupAB, root + "/dirAB700", "700", 200, 200 );
+ createDir( userA, passA, groupAB, root + "/dirAB770", "770", 200, 200 );
+ createDir( userA, passA, groupAB, root + "/dirAB707", "707", 200, 200 );
+ createDir( userA, passA, groupAB, root + "/dirAB777", "777", 200, 200 );
+
+ // CREATE: Files
+ // userA:groupA
+ createFile( userA, passA, groupA, root + "/dirA700/fileA700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupA, root + "/dirA770/fileA770", "770", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupA, root + "/dirA707/fileA707", "707", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupA, root + "/dirA777/fileA777", "777", "text/plain", "small1.txt", 307, 201, 200 );
+ // userA:groupAB
+ createFile( userA, passA, groupAB, root + "/dirAB700/fileAB700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupAB, root + "/dirAB770/fileAB770", "770", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupAB, root + "/dirAB707/fileAB707", "707", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userA, passA, groupAB, root + "/dirAB777/fileAB777", "777", "text/plain", "small1.txt", 307, 201, 200 );
+ // userB:groupB
+ createFile( userB, passB, groupB, root + "/dirA700/fileB700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+ createFile( userB, passB, groupB, root + "/dirA770/fileB700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+//kam:20130219[ chmod seems to be broken at least in Sandbox 1.2
+// createFile( userB, passB, groupB, root + "/dirA707/fileB700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+// createFile( userB, passB, groupB, root + "/dirA777/fileB700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+//kam]
+ // userB:groupAB
+ createFile( userB, passB, groupAB, root + "/dirA700/fileBA700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+ createFile( userB, passB, groupAB, root + "/dirA770/fileBA700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+ createFile( userB, passB, groupAB, root + "/dirA707/fileBA700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+ createFile( userB, passB, groupAB, root + "/dirA777/fileBA700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+ // userC:groupC
+ createFile( userC, passC, groupC, root + "/dirA700/fileC700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+ createFile( userC, passC, groupC, root + "/dirA770/fileC700", "700", "text/plain", "small1.txt", 307, 403, 0 );
+//kam:20130219[ chmod seems to be broken at least in Sandbox 1.2
+// createFile( userC, passC, groupC, root + "/dirA707/fileC700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+// createFile( userC, passC, groupC, root + "/dirA777/fileC700", "700", "text/plain", "small1.txt", 307, 201, 200 );
+//kam]
+
+ // READ
+ // userA
+ readFile( userA, passA, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ readFile( userA, passA, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ readFile( userA, passA, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ readFile( userA, passA, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ // userB:groupB
+ readFile( userB, passB, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userB, passB, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userB, passB, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ readFile( userB, passB, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ // userB:groupAB
+ readFile( userB, passB, root + "/dirAB700/fileAB700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userB, passB, root + "/dirAB770/fileAB770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userB, passB, root + "/dirAB707/fileAB707", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userB, passB, root + "/dirAB777/fileAB777", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ // userC:groupC
+ readFile( userC, passC, root + "/dirA700/fileA700", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userC, passC, root + "/dirA770/fileA770", "text/plain", "small1.txt", HttpStatus.SC_FORBIDDEN );
+ readFile( userC, passC, root + "/dirA707/fileA707", "text/plain", "small1.txt", HttpStatus.SC_OK );
+ readFile( userC, passC, root + "/dirA777/fileA777", "text/plain", "small1.txt", HttpStatus.SC_OK );
+
+ //NEGATIVE: Test a bad password.
+ if( driver.isUseGateway() ) {
+ given()
+ //.log().all()
+ .auth().preemptive().basic( userA, "invalid-password" )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .queryParam( "op", "OPEN" )
+ .expect()
+ //.log().all()
+ .statusCode( HttpStatus.SC_UNAUTHORIZED )
+ .when().get( driver.getUrl("WEBHDFS") + "/v1" + root + "/dirA700/fileA700" );
+ }
+ driver.assertComplete();
+
+ // UPDATE (Negative First)
+ updateFile( userC, passC, root + "/dirA700/fileA700", "text/plain", "small2.txt", 307, 403 );
+ updateFile( userB, passB, root + "/dirAB700/fileAB700", "text/plain", "small2.txt", 307, 403 );
+ updateFile( userB, passB, root + "/dirAB770/fileAB700", "text/plain", "small2.txt", 307, 403 );
+ updateFile( userB, passB, root + "/dirAB770/fileAB770", "text/plain", "small2.txt", 307, 403 );
+ updateFile( userA, passA, root + "/dirA700/fileA700", "text/plain", "small2.txt", 307, 201 );
+
+ // DELETE (Negative First)
+ deleteFile( userC, passC, root + "/dirA700/fileA700", "false", HttpStatus.SC_FORBIDDEN );
+ deleteFile( userB, passB, root + "/dirAB700/fileAB700", "false", HttpStatus.SC_FORBIDDEN );
+ deleteFile( userB, passB, root + "/dirAB770/fileAB770", "false", HttpStatus.SC_FORBIDDEN );
+ deleteFile( userA, passA, root + "/dirA700/fileA700", "false", HttpStatus.SC_OK );
+
+ // Cleanup anything that might have been leftover because the test failed previously.
+ deleteFile( userA, passA, root, "true", HttpStatus.SC_OK );
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testJavaMapReduceViaWebHCat() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testJavaMapReduceViaWebHCat";
+ String user = "mapred";
+ String pass = "mapred-password";
+ String group = "mapred";
+// String user = "hcat";
+// String pass = "hcat-password";
+// String group = "hcat";
+
+ // Cleanup anything that might have been leftover because the test failed previously.
+ deleteFile( user, pass, root, "true", HttpStatus.SC_OK );
+
+ /* Put the mapreduce code into HDFS. (hadoop-examples.jar)
+ curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE'
+ */
+ createFile( user, pass, null, root+"/hadoop-examples.jar", "777", "application/octet-stream", findHadoopExamplesJar(), 307, 201, 200 );
+
+ /* Put the data file into HDFS (changes.txt)
+ curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE'
+ */
+ createFile( user, pass, null, root+"/input/changes.txt", "777", "text/plain", "changes.txt", 307, 201, 200 );
+
+ /* Create the output directory
+ curl -X PUT 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/output?op=MKDIRS&user.name=hdfs'
+ */
+ createDir( user, pass, null, root+"/output", "777", 200, 200 );
+
+ /* Submit the job
+ curl -d user.name=hdfs -d jar=wordcount/hadoop-examples.jar -d class=org.apache.org.apache.hadoop.examples.WordCount -d arg=wordcount/input -d arg=wordcount/output 'http://localhost:8888/org.apache.org.apache.knox.gateway/cluster/templeton/v1/mapreduce/jar'
+ {"id":"job_201210301335_0059"}
+ */
+ String job = submitJava(
+ user, pass,
+ root+"/hadoop-examples.jar", "org.apache.org.apache.hadoop.examples.WordCount",
+ root+"/input", root+"/output",
+ 200 );
+
+ /* Get the job status
+ curl 'http://vm:50111/templeton/v1/queue/:jobid?user.name=hdfs'
+ */
+ queryQueue( user, pass, job );
+
+ // Can't really check for the output here because the job won't be done.
+ /* Retrieve results
+ curl 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input?op=LISTSTATUS'
+ */
+
+ if( CLEANUP_TEST ) {
+ // Cleanup anything that might have been leftover because the test failed previously.
+ deleteFile( user, pass, root, "true", HttpStatus.SC_OK );
+ }
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testPigViaWebHCat() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayWebHCatFuncTest/testPigViaWebHCat";
+ String user = "mapred";
+ String pass = "mapred-password";
+ String group = "mapred";
+
+ // Cleanup if previous run failed.
+ deleteFile( user, pass, root, "true", 200, 404 );
+
+ // Post the data to HDFS
+ createFile( user, pass, null, root + "/passwd.txt", "777", "text/plain", "passwd.txt", 307, 201, 200 );
+
+ // Post the script to HDFS
+ createFile( user, pass, null, root+"/script.pig", "777", "text/plain", "script.pig", 307, 201, 200 );
+
+ // Create the output directory
+ createDir( user, pass, null, root + "/output", "777", 200, 200 );
+
+ // Submit the job
+ submitPig( user, pass, group, root + "/script.pig", "-v", root + "/output", 200 );
+
+ // Check job status (if possible)
+ // Check output (if possible)
+
+ // Cleanup
+ deleteFile( user, pass, root, "true", 200 );
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHiveViaWebHCat() throws IOException {
+ LOG_ENTER();
+ String user = "hive";
+ String pass = "hive-password";
+ String group = "hive";
+ String root = "/tmp/GatewayWebHCatFuncTest/testHiveViaWebHCat";
+
+ // Cleanup if previous run failed.
+ deleteFile( user, pass, root, "true", 200, 404 );
+
+ // Post the data to HDFS
+
+ // Post the script to HDFS
+ createFile(user, pass, null, root + "/script.hive", "777", "text/plain", "script.hive", 307, 201, 200);
+
+ // Submit the job
+ submitHive(user, pass, group, root + "/script.hive", root + "/output", 200);
+
+ // Check job status (if possible)
+ // Check output (if possible)
+
+ // Cleanup
+ deleteFile( user, pass, root, "true", 200 );
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testOozieJobSubmission() throws Exception {
+ LOG_ENTER();
+ String root = "/tmp/GatewayBasicFuncTest/testOozieJobSubmission";
+ String user = "hdfs";
+ String pass = "hdfs-password";
+ String group = "hdfs";
+
+ // Cleanup anything that might have been leftover because the test failed previously.
+ deleteFile( user, pass, root, "true", HttpStatus.SC_OK );
+
+ /* Put the workflow definition into HDFS */
+ createFile( user, pass, group, root+"/workflow.xml", "666", "application/octet-stream", "oozie-workflow.xml", 307, 201, 200 );
+
+ /* Put the mapreduce code into HDFS. (hadoop-examples.jar)
+ curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE'
+ */
+ createFile( user, pass, group, root+"/lib/hadoop-examples.jar", "777", "application/octet-stream", findHadoopExamplesJar(), 307, 201, 200 );
+
+ /* Put the data file into HDFS (changes.txt)
+ curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.knox.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE'
+ */
+ createFile( user, pass, group, root+"/input/changes.txt", "666", "text/plain", "changes.txt", 307, 201, 200 );
+
+ VelocityEngine velocity = new VelocityEngine();
+ velocity.setProperty( RuntimeConstants.RUNTIME_LOG_LOGSYSTEM_CLASS, "org.apache.velocity.runtime.log.NullLogSystem" );
+ velocity.setProperty( RuntimeConstants.RESOURCE_LOADER, "classpath" );
+ velocity.setProperty( "classpath.resource.loader.class", ClasspathResourceLoader.class.getName() );
+ velocity.init();
+
+ VelocityContext context = new VelocityContext();
+ context.put( "userName", user );
+ context.put( "nameNode", "hdfs://sandbox:8020" );
+ context.put( "jobTracker", "sandbox:50300" );
+ //context.put( "appPath", "hdfs://sandbox:8020" + root );
+ context.put( "appPath", root );
+ context.put( "inputDir", root + "/input" );
+ context.put( "outputDir", root + "/output" );
+
+ //URL url = TestUtils.getResourceUrl( GatewayBasicFuncTest.class, "oozie-jobs-submit-request.xml" );
+ //String name = url.toExternalForm();
+ String name = TestUtils.getResourceName( this.getClass(), "oozie-jobs-submit-request.xml" );
+ Template template = velocity.getTemplate( name );
+ StringWriter sw = new StringWriter();
+ template.merge( context, sw );
+ String request = sw.toString();
+ //System.out.println( "REQUEST=" + request );
+
+ /* Submit the job via Oozie. */
+ String id = oozieSubmitJob( user, pass, request, 201 );
+ //System.out.println( "ID=" + id );
+
+ String success = "SUCCEEDED";
+ String status = "UNKNOWN";
+ long delay = 1000 * 1; // 1 second.
+ long limit = 1000 * 60; // 60 seconds.
+ long start = System.currentTimeMillis();
+ while( System.currentTimeMillis() <= start+limit ) {
+ status = oozieQueryJobStatus( user, pass, id, 200 );
+ //System.out.println( "Status=" + status );
+ if( success.equalsIgnoreCase( status ) ) {
+ break;
+ } else {
+ //System.out.println( "Status=" + status );
+ Thread.sleep( delay );
+ }
+ }
+ //System.out.println( "Status is " + status + " after " + ((System.currentTimeMillis()-start)/1000) + " seconds." );
+ MatcherAssert.assertThat( status, is( success ) );
+
+ if( CLEANUP_TEST ) {
+ // Cleanup anything that might have been leftover because the test failed previously.
+ deleteFile( user, pass, root, "true", HttpStatus.SC_OK );
+ }
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testBasicHiveJDBCUseCase() throws IOException {
+ LOG_ENTER();
+ String root = "/tmp/GatewayHiveJDBCFuncTest/testBasicHiveUseCase";
+ String username = "hive";
+ String password = "hive-password";
+ InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0];
+
+ // This use case emulates simple JDBC scenario which consists of following steps:
+ // -open connection;
+ // -configure Hive using 'execute' statements (this also includes execution of 'close operation' requests internally);
+ // -execution of create table command;
+ // -execution of select from table command;
+ // Data insertion is omitted because it causes a lot of additional command during insertion/querying.
+ // All binary data was intercepted during real scenario and stored into files as array of bytes.
+
+ // open session
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/open-session-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/open-session-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ Response response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/open-session-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/open-session-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/open-session-result.bin" ) ) );
+
+ driver.assertComplete();
+
+ // execute 'set hive.fetch.output.serde=...' (is called internally be JDBC driver)
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-fetch-output-serde-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'set hive.fetch.output.serde=...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-1-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-1-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-1-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'set hive.server2.http.path=...' (is called internally be JDBC driver)
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-set-server2-http-path-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-server2-http-path-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'set hive.server2.http.path=...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-2-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-2-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-2-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'set hive.server2.servermode=...' (is called internally be JDBC driver)
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-set-server2-servermode-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-server2-servermode-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'set hive.server2.servermode=...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-3-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-3-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-3-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'set hive.security.authorization.enabled=...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-set-security-authorization-enabled-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'set hive.security.authorization.enabled=...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-4-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-4-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-4-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'create table...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-create-table-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-create-table-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-create-table-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'create table...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-5-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-5-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-5-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'select * from...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/execute-select-from-table-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/execute-select-from-table-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/execute-select-from-table-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'GetResultSetMetadata' (is called internally be JDBC driver)
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/get-result-set-metadata-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/get-result-set-metadata-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/get-result-set-metadata-result.bin" ) ) );
+ driver.assertComplete();
+
+ // execute 'FetchResults' (is called internally be JDBC driver)
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/fetch-results-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/fetch-results-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/fetch-results-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/fetch-results-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/fetch-results-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close operation for execute 'select * from...'
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-operation-6-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-operation-6-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-operation-6-result.bin" ) ) );
+ driver.assertComplete();
+
+ // close session
+ driver.getMock( "HIVE" )
+ .expect()
+ .method( "POST" )
+ .content( driver.getResourceBytes( "hive/close-session-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .respond()
+ .characterEncoding( "UTF-8" )
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( "hive/close-session-result.bin" ) )
+ .contentType( "application/x-thrift" );
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .content( driver.getResourceBytes( "hive/close-session-request.bin" ) )
+ .contentType( "application/x-thrift" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( "hive/close-session-result.bin" ) ) )
+ .contentType( "application/x-thrift" )
+ .when().post( driver.getUrl( "HIVE" ) );
+ assertThat( response.body().asByteArray(), is( driver.getResourceBytes( "hive/close-session-result.bin" ) ) );
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseGetTableList() throws IOException {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+ String resourceName = "hbase/table-list";
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/" )
+ .header( "Accept", ContentType.XML.toString() )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() );
+
+ Response response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", ContentType.XML.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( ContentType.XML )
+ .when().get( driver.getUrl( "WEBHBASE" ) );
+
+ MatcherAssert
+ .assertThat(
+ the( response.getBody().asString() ),
+ isEquivalentTo( the( driver.getResourceString( resourceName + ".xml", UTF8 ) ) ) );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/" )
+ .header( "Accept", ContentType.JSON.toString() )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resourceName + ".json" ) )
+ .contentType( ContentType.JSON.toString() );
+
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", ContentType.JSON.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( ContentType.JSON )
+ .when().get( driver.getUrl( "WEBHBASE" ) );
+
+ MatcherAssert
+ .assertThat( response.getBody().asString(), sameJSONAs( driver.getResourceString( resourceName + ".json", UTF8 ) ) );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( "/" )
+ .header( "Accept", "application/x-protobuf" )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceString( resourceName + ".protobuf", UTF8 ), UTF8 )
+ .contentType( "application/x-protobuf" );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", "application/x-protobuf" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( "application/x-protobuf" )
+ .content( is( driver.getResourceString( resourceName + ".protobuf", UTF8 ) ) )
+ .when().get( driver.getUrl( "WEBHBASE" ) );
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseCreateTableAndVerifySchema() throws IOException {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+ String resourceName = "hbase/table-schema";
+ String path = "/table/schema";
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( path )
+ .respond()
+ .status( HttpStatus.SC_CREATED )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() )
+ .header( "Location", driver.getRealUrl( "WEBHBASE" ) + path );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode( HttpStatus.SC_CREATED )
+ .contentType( ContentType.XML )
+ .header( "Location", startsWith( driver.getUrl( "WEBHBASE" ) + path ) )
+ .when().put(driver.getUrl("WEBHBASE") + path);
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( path )
+ .respond()
+ .status(HttpStatus.SC_CREATED)
+ .content(driver.getResourceBytes(resourceName + ".json"))
+ .contentType(ContentType.JSON.toString())
+ .header("Location", driver.getRealUrl("WEBHBASE") + path);
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode( HttpStatus.SC_CREATED )
+ .contentType( ContentType.JSON )
+ .header( "Location", startsWith( driver.getUrl( "WEBHBASE" ) + path ) )
+ .when().put( driver.getUrl( "WEBHBASE" ) + path );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( path )
+ .respond()
+ .status( HttpStatus.SC_CREATED )
+ .content( driver.getResourceBytes( resourceName + ".protobuf" ) )
+ .contentType( "application/x-protobuf" )
+ .header("Location", driver.getRealUrl("WEBHBASE") + path);
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode(HttpStatus.SC_CREATED)
+ .contentType("application/x-protobuf")
+ .header("Location", startsWith(driver.getUrl("WEBHBASE") + path))
+ .when().put(driver.getUrl("WEBHBASE") + path);
+ driver.assertComplete();
+
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseGetTableSchema() throws IOException {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+ String resourceName = "hbase/table-metadata";
+ String path = "/table/schema";
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( path )
+ .header("Accept", ContentType.XML.toString())
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes(resourceName + ".xml"))
+ .contentType(ContentType.XML.toString());
+
+ Response response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", ContentType.XML.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( ContentType.XML )
+ .when().get( driver.getUrl( "WEBHBASE" ) + path );
+
+ MatcherAssert
+ .assertThat(
+ the(response.getBody().asString()),
+ isEquivalentTo(the(driver.getResourceString(resourceName + ".xml", UTF8))));
+ driver.assertComplete();
+
+ driver.getMock("WEBHBASE")
+ .expect()
+ .method("GET")
+ .pathInfo(path)
+ .header("Accept", ContentType.JSON.toString())
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes(resourceName + ".json"))
+ .contentType(ContentType.JSON.toString());
+
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", ContentType.JSON.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( ContentType.JSON )
+ .when().get( driver.getUrl( "WEBHBASE" ) + path );
+
+ MatcherAssert
+ .assertThat(response.getBody().asString(), sameJSONAs(driver.getResourceString(resourceName + ".json", UTF8)));
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( path )
+ .header( "Accept", "application/x-protobuf" )
+ .respond()
+ .status( HttpStatus.SC_OK )
+ .content( driver.getResourceBytes( resourceName + ".protobuf" ) )
+ .contentType("application/x-protobuf");
+
+ response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", "application/x-protobuf" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ //.content( is( driver.getResourceBytes( resourceName + ".protobuf" ) ) )
+ .contentType( "application/x-protobuf" )
+ .when().get( driver.getUrl( "WEBHBASE" ) + path );
+ // RestAssured seems to be screwing up the binary comparison so do it explicitly.
+ assertThat( driver.getResourceBytes( resourceName + ".protobuf" ), is( response.body().asByteArray() ) );
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseInsertDataIntoTable() throws IOException {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+
+ String resourceName = "hbase/table-data";
+ String singleRowPath = "/table/testrow";
+ String multipleRowPath = "/table/false-row-key";
+
+ //PUT request
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( multipleRowPath )
+ //.header( "Content-Type", ContentType.XML.toString() )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() )
+ .respond()
+ .status(HttpStatus.SC_OK);
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", ContentType.XML.toString() )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().put(driver.getUrl("WEBHBASE") + multipleRowPath);
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "PUT" )
+ .pathInfo( singleRowPath )
+ //.header( "Content-Type", ContentType.JSON.toString() )
+ .contentType( ContentType.JSON.toString() )
+ .respond()
+ .status( HttpStatus.SC_OK );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", ContentType.JSON.toString() )
+ .content( driver.getResourceBytes( resourceName + ".json" ) )
+ .contentType( ContentType.JSON.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().put(driver.getUrl("WEBHBASE") + singleRowPath);
+ driver.assertComplete();
+
+ driver.getMock("WEBHBASE")
+ .expect()
+ .method("PUT")
+ .pathInfo(multipleRowPath)
+ //.header( "Content-Type", "application/x-protobuf" )
+ .contentType("application/x-protobuf")
+ .content(driver.getResourceBytes(resourceName + ".protobuf"))
+ .respond()
+ .status(HttpStatus.SC_OK);
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", "application/x-protobuf" )
+ .content( driver.getResourceBytes( resourceName + ".protobuf" ) )
+ .contentType( "application/x-protobuf" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().put( driver.getUrl( "WEBHBASE" ) + multipleRowPath );
+ driver.assertComplete();
+
+ //POST request
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "POST" )
+ .pathInfo( multipleRowPath )
+ //.header( "Content-Type", ContentType.XML.toString() )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() )
+ .respond()
+ .status( HttpStatus.SC_OK );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", ContentType.XML.toString() )
+ .content( driver.getResourceBytes( resourceName + ".xml" ) )
+ .contentType( ContentType.XML.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().post( driver.getUrl( "WEBHBASE" ) + multipleRowPath );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "POST" )
+ .pathInfo( singleRowPath )
+ //.header( "Content-Type", ContentType.JSON.toString() )
+ .contentType( ContentType.JSON.toString() )
+ .respond()
+ .status( HttpStatus.SC_OK );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", ContentType.JSON.toString() )
+ .content( driver.getResourceBytes( resourceName + ".json" ) )
+ .contentType( ContentType.JSON.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().post( driver.getUrl( "WEBHBASE" ) + singleRowPath );
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "POST" )
+ .pathInfo( multipleRowPath )
+ //.header( "Content-Type", "application/x-protobuf" )
+ .content( driver.getResourceBytes( resourceName + ".protobuf" ) )
+ .contentType( "application/x-protobuf" )
+ .respond()
+ .status( HttpStatus.SC_OK );
+
+ given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ //.header( "Content-Type", "application/x-protobuf" )
+ .content( driver.getResourceBytes( resourceName + ".protobuf" ) )
+ .contentType( "application/x-protobuf" )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().post(driver.getUrl("WEBHBASE") + multipleRowPath);
+ driver.assertComplete();
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseDeleteDataFromTable() {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+
+ String tableId = "table";
+ String rowId = "row";
+ String familyId = "family";
+ String columnId = "column";
+
+ driver.getMock("WEBHBASE")
+ .expect()
+ .from("testHBaseDeleteDataFromTable-1")
+ .method("DELETE")
+ .pathInfo("/" + tableId + "/" + rowId)
+ .respond()
+ .status(HttpStatus.SC_OK);
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().delete(driver.getUrl("WEBHBASE") + "/" + tableId + "/" + rowId);
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .from("testHBaseDeleteDataFromTable-2")
+ .method("DELETE")
+ .pathInfo("/" + tableId + "/" + rowId + "/" + familyId)
+ .respond()
+ .status( HttpStatus.SC_OK );
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().delete(driver.getUrl("WEBHBASE") + "/" + tableId + "/" + rowId + "/" + familyId);
+ driver.assertComplete();
+
+ driver.getMock("WEBHBASE")
+ .expect()
+ .from("testHBaseDeleteDataFromTable-3")
+ .method("DELETE")
+ .pathInfo("/" + tableId + "/" + rowId + "/" + familyId + ":" + columnId)
+ .respond()
+ .status(HttpStatus.SC_OK);
+
+ given()
+ .auth().preemptive().basic(username, password)
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .when().delete(driver.getUrl("WEBHBASE") + "/" + tableId + "/" + rowId + "/" + familyId + ":" + columnId);
+ driver.assertComplete();
+
+ LOG_EXIT();
+ }
+
+ @Test( timeout = TestUtils.MEDIUM_TIMEOUT )
+ public void testHBaseQueryTableData() throws IOException {
+ LOG_ENTER();
+ String username = "hbase";
+ String password = "hbase-password";
+
+ String resourceName = "hbase/table-data";
+
+ String allRowsPath = "/table/*";
+ String rowsStartsWithPath = "/table/row*";
+ String rowsWithKeyPath = "/table/row";
+ String rowsWithKeyAndColumnPath = "/table/row/family:col";
+
+ driver.getMock("WEBHBASE")
+ .expect()
+ .method("GET")
+ .pathInfo(allRowsPath)
+ .header("Accept", ContentType.XML.toString())
+ .respond()
+ .status(HttpStatus.SC_OK)
+ .content(driver.getResourceBytes(resourceName + ".xml"))
+ .contentType(ContentType.XML.toString());
+
+ Response response = given()
+ .auth().preemptive().basic( username, password )
+ .header("X-XSRF-Header", "jksdhfkhdsf")
+ .header( "Accept", ContentType.XML.toString() )
+ .expect()
+ .statusCode( HttpStatus.SC_OK )
+ .contentType( ContentType.XML )
+ .when().get( driver.getUrl( "WEBHBASE" ) + allRowsPath );
+
+ MatcherAssert
+ .assertThat(
+ the(response.getBody().asString()),
+ isEquivalentTo(the(driver.getResourceString(resourceName + ".xml", UTF8))));
+ driver.assertComplete();
+
+ driver.getMock( "WEBHBASE" )
+ .expect()
+ .method( "GET" )
+ .pathInfo( rowsStartsWithPath
<TRUNCATED>