You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2013/03/15 18:38:50 UTC
[3/3] git commit: Client groovy examples all working. Including SSL.
Updated Branches:
refs/heads/master ba9dc3e5f -> fe50e9b2f
Client groovy examples all working. Including SSL.
Project: http://git-wip-us.apache.org/repos/asf/incubator-knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-knox/commit/37b6d21d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-knox/tree/37b6d21d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-knox/diff/37b6d21d
Branch: refs/heads/master
Commit: 37b6d21d0eb6a6ec7110ab36821803f297995672
Parents: ba9dc3e
Author: Kevin Minder <ke...@hortonworks.com>
Authored: Fri Mar 15 12:42:11 2013 -0400
Committer: Kevin Minder <ke...@hortonworks.com>
Committed: Fri Mar 15 13:36:21 2013 -0400
----------------------------------------------------------------------
gateway-release/README | 6 +-
gateway-shell/pom.xml | 29 +++-
.../hadoop/gateway/shell/AbstractRequest.java | 34 +++-
.../hadoop/gateway/shell/AbstractResponse.java | 68 +++++++-
.../org/apache/hadoop/gateway/shell/Example.groovy | 39 +++--
.../org/apache/hadoop/gateway/shell/Hadoop.java | 137 +++++++++++++++
.../hadoop/gateway/shell/HadoopException.java | 30 +++
.../org/apache/hadoop/gateway/shell/KnoxShell.java | 48 -----
.../org/apache/hadoop/gateway/shell/Shell.java | 48 +++++
.../apache/hadoop/gateway/shell/hadoop/Hadoop.java | 51 ------
.../hadoop/gateway/shell/hdfs/Example.groovy | 40 +++++
.../org/apache/hadoop/gateway/shell/hdfs/Get.java | 78 ++++++++
.../org/apache/hadoop/gateway/shell/hdfs/Hdfs.java | 22 ++-
.../hadoop/gateway/shell/hdfs/HdfsLsRequest.java | 41 -----
.../hadoop/gateway/shell/hdfs/HdfsLsResponse.java | 33 ----
.../gateway/shell/hdfs/HdfsMkdirRequest.java | 46 -----
.../gateway/shell/hdfs/HdfsMkdirResponse.java | 29 ---
.../hadoop/gateway/shell/hdfs/HdfsPutRequest.java | 57 ------
.../hadoop/gateway/shell/hdfs/HdfsPutResponse.java | 29 ---
.../hadoop/gateway/shell/hdfs/HdfsRmRequest.java | 50 ------
.../hadoop/gateway/shell/hdfs/HdfsRmResponse.java | 29 ---
.../org/apache/hadoop/gateway/shell/hdfs/Ls.java | 63 +++++++
.../apache/hadoop/gateway/shell/hdfs/Mkdir.java | 69 ++++++++
.../org/apache/hadoop/gateway/shell/hdfs/Put.java | 101 +++++++++++
.../org/apache/hadoop/gateway/shell/hdfs/Rm.java | 74 ++++++++
.../hadoop/gateway/shell/job/ExampleHive.groovy | 54 ++++++
.../hadoop/gateway/shell/job/ExampleJava.groovy | 56 ++++++
.../hadoop/gateway/shell/job/ExamplePig.groovy | 58 ++++++
.../org/apache/hadoop/gateway/shell/job/Hive.java | 93 ++++++++++
.../org/apache/hadoop/gateway/shell/job/Java.java | 96 ++++++++++
.../org/apache/hadoop/gateway/shell/job/Job.java | 22 ++-
.../hadoop/gateway/shell/job/JobJavaRequest.java | 58 ------
.../hadoop/gateway/shell/job/JobJavaResponse.java | 44 -----
.../org/apache/hadoop/gateway/shell/job/Pig.java | 95 ++++++++++
.../org/apache/hadoop/gateway/shell/job/Queue.java | 54 ++++++
.../apache/hadoop/gateway/shell/job/Status.java | 61 +++++++
.../hadoop/gateway/shell/workflow/Example.groovy | 86 +++++++++
.../hadoop/gateway/shell/workflow/Status.java | 61 +++++++
.../hadoop/gateway/shell/workflow/Submit.java | 91 ++++++++++
.../hadoop/gateway/shell/workflow/Workflow.java | 12 +-
.../shell/workflow/WorkflowSubmitRequest.java | 33 ----
.../shell/workflow/WorkflowSubmitResponse.java | 29 ---
pom.xml | 6 +
43 files changed, 1629 insertions(+), 631 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-release/README
----------------------------------------------------------------------
diff --git a/gateway-release/README b/gateway-release/README
index d458d5a..ccced99 100644
--- a/gateway-release/README
+++ b/gateway-release/README
@@ -273,7 +273,7 @@ curl -i -k -u mapred:mapred-password -T README -X PUT '{Value of Location header
# 7. Submit the word count job via WebHCat/Templeton.
# Take note of the Job ID in the JSON response as this will be used in the next step.
-curl -i -k -u mapred:mapred-password -X POST \
+curl -v -i -k -u mapred:mapred-password -X POST \
-d jar=/tmp/test/hadoop-examples.jar -d class=wordcount \
-d arg=/tmp/test/input -d arg=/tmp/test/output \
'https://localhost:8443/gateway/sample/templeton/api/v1/mapreduce/jar'
@@ -344,8 +344,8 @@ sed -e s/REPLACE.NAMENODE.RPCHOSTPORT/{NameNode host:port}/ \
# 8. Submit the job via Oozie
# Take note of the Job ID in the JSON response as this will be used in the next step.
-curl -i -k -u mapred:mapred-password -T workflow-configuration.xml -X POST \
- 'https://localhost:8443/gateway/oozie/api/v1/jobs?action=start'
+curl -i -k -u mapred:mapred-password -T workflow-configuration.xml -H Content-Type:application/xml -X POST \
+ 'https://localhost:8443/gateway/oozie/sample/api/v1/jobs?action=start'
# 9. Query the job status via Oozie.
curl -i -k -u mapred:mapred-password -X GET \
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-shell/pom.xml b/gateway-shell/pom.xml
index 8e868e2..e52f2db 100644
--- a/gateway-shell/pom.xml
+++ b/gateway-shell/pom.xml
@@ -43,7 +43,7 @@
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
- <Main-Class>org.apache.hadoop.gateway.shell.KnoxShell</Main-Class>
+ <Main-Class>org.apache.hadoop.gateway.shell.Shell</Main-Class>
</manifestEntries>
</transformer>
</transformers>
@@ -79,21 +79,38 @@
<version>1.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path</artifactId>
+ <version>0.8.1</version>
+ </dependency>
+ <!--
+ <dependency>
<groupId>com.jayway.restassured</groupId>
<artifactId>rest-assured</artifactId>
<version>1.7.2</version>
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>2.1</version>
- </dependency>
- <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>1.2.1</version>
</dependency>
+ -->
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.1</version>
+ </dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractRequest.java
index 429dcf9..f850e39 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractRequest.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractRequest.java
@@ -1,9 +1,15 @@
package org.apache.hadoop.gateway.shell;
-import com.jayway.restassured.specification.RequestSpecification;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.message.BasicNameValuePair;
-import static com.jayway.restassured.RestAssured.with;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.List;
/**
* Licensed to the Apache Software Foundation (ASF) under one
@@ -25,19 +31,33 @@ import static com.jayway.restassured.RestAssured.with;
public abstract class AbstractRequest {
private Hadoop hadoop;
- private RequestSpecification request;
public AbstractRequest( Hadoop hadoop ) {
this.hadoop = hadoop;
- request = with().spec( hadoop.request() );
}
protected Hadoop hadoop() {
return hadoop;
}
- protected RequestSpecification request() {
- return request;
+ protected HttpResponse execute( HttpRequest request ) throws IOException {
+ return hadoop.execute( request );
+ }
+
+ protected URIBuilder uri( String... parts ) throws URISyntaxException {
+ return new URIBuilder( hadoop.base() + StringUtils.join( parts ) );
+ }
+
+ protected void addQueryParam( URIBuilder uri, String name, Object value ) {
+ if( value != null ) {
+ uri.addParameter( name, value.toString() );
+ }
+ }
+
+ protected void addParam( List<NameValuePair> list, String name, String value ) {
+ if( value != null ) {
+ list.add( new BasicNameValuePair( name, value ) );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractResponse.java
index 645f8d4..1028fff 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractResponse.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/AbstractResponse.java
@@ -1,6 +1,11 @@
package org.apache.hadoop.gateway.shell;
-import com.jayway.restassured.response.Response;
+import org.apache.http.HttpResponse;
+import org.apache.http.entity.ContentType;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
/**
* Licensed to the Apache Software Foundation (ASF) under one
@@ -21,14 +26,69 @@ import com.jayway.restassured.response.Response;
*/
public abstract class AbstractResponse {
- private Response response;
+ private HttpResponse response;
+ private boolean consumed = false;
+ private String string;
+ private InputStream stream;
+ private byte[] bytes;
- protected AbstractResponse( Response response ) {
+ protected AbstractResponse( HttpResponse response ) {
this.response = response;
}
- protected Response response() {
+ public void consume() {
+ if( !consumed ) {
+ EntityUtils.consumeQuietly( response.getEntity() );
+ consumed = true;
+ }
+ }
+
+ public void close() {
+ consume();
+ }
+
+ protected HttpResponse response() {
return response;
}
+ public boolean isConsumed() {
+ return consumed;
+ }
+
+ public long getContentLength() {
+ return response.getEntity().getContentLength();
+ }
+
+ public String getContentType() {
+ return ContentType.getOrDefault( response.getEntity() ).getMimeType();
+ }
+
+ public String getContentEncoding() {
+ return ContentType.getOrDefault( response.getEntity() ).getCharset().name();
+ }
+
+ public InputStream getStream() throws IOException {
+ if( !consumed && stream == null ) {
+ stream = response.getEntity().getContent();
+ consumed = true;
+ }
+ return stream;
+ }
+
+ public String getString() throws IOException {
+ if( !consumed && string == null ) {
+ string = EntityUtils.toString( response.getEntity() );
+ consumed = true;
+ }
+ return string;
+ }
+
+ public byte[] getBytes() throws IOException {
+ if( !consumed && bytes == null ) {
+ bytes = EntityUtils.toByteArray( response.getEntity() );
+ consumed = true;
+ }
+ return bytes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Example.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Example.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Example.groovy
index 6d1b53e..f8f1a1d 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Example.groovy
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Example.groovy
@@ -1,9 +1,3 @@
-package org.apache.hadoop.gateway.shell
-
-import org.apache.hadoop.gateway.shell.hdfs.Hdfs
-import org.apache.hadoop.gateway.shell.job.Job
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop
-
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -21,17 +15,26 @@ import org.apache.hadoop.gateway.shell.hadoop.Hadoop
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-hadoop = Hadoop.login( "http://localhost:8443/gateway/sample", "mapred", "mapred-password" )
-println Hdfs.ls(hadoop).dir( "/" ).go().json
-Hdfs.rm(hadoop).file( "/tmp/test" ).recursive().go()
-Hdfs.mkdir(hadoop).dir( "/tmp/test").go()
-Hdfs.put(hadoop).from("LICENSE").to("/tmp/test/input/LICENSE").go()
-Hdfs.put(hadoop).from("hadoop-examples.jar").to("/tmp/test/hadoop-examples.jar").go()
-println Job.java(hadoop) \
- .jar("/tmp/test/hadoop-examples.jar") \
- .app("wordcount") \
- .input("/temp/test/input") \
- .output("/temp/test/output") \
- .go().jobId
+package org.apache.hadoop.gateway.shell
+
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+import org.apache.hadoop.gateway.shell.job.Job as job
+
+hadoop = Hadoop.login( "https://localhost:8443/gateway/sample", "mapred", "mapred-password" )
+
+println Hdfs.ls(hadoop).dir( "/" ).now().asString
+
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+
+hdfs.mkdir(hadoop).dir( "/tmp/test").now()
+
+hdfs.put(hadoop).file("LICENSE").to("/tmp/test/input/LICENSE").now()
+hdfs.put(hadoop).file("hadoop-examples.jar").to("/tmp/test/hadoop-examples.jar").now()
+println job.submitJava(hadoop) \
+ .jar("/tmp/test/hadoop-examples.jar") \
+ .app("wordcount") \
+ .input("/temp/test/input") \
+ .output("/temp/test/output") \
+ .now().jobId
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Hadoop.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Hadoop.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Hadoop.java
new file mode 100644
index 0000000..47baf8c
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Hadoop.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell;
+
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.AuthCache;
+import org.apache.http.client.protocol.ClientContext;
+import org.apache.http.conn.scheme.PlainSocketFactory;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.auth.BasicScheme;
+import org.apache.http.impl.client.BasicAuthCache;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.protocol.BasicHttpContext;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.GeneralSecurityException;
+
+public class Hadoop {
+
+ String base;
+ HttpHost host;
+ DefaultHttpClient client;
+ BasicHttpContext context;
+ String username;
+ String password;
+
+ public static Hadoop login( String url, String username, String password ) throws URISyntaxException {
+ return new Hadoop( url, username, password );
+ }
+
+ private Hadoop( String url, String username, String password ) throws HadoopException, URISyntaxException {
+ this.base = url;
+ this.username = username;
+ this.password = password;
+
+ URI uri = new URI( url );
+ host = new HttpHost( uri.getHost(), uri.getPort(), uri.getScheme() );
+
+ try {
+ client = createClient();
+ client.getCredentialsProvider().setCredentials(
+ new AuthScope( host.getHostName(), host.getPort() ),
+ new UsernamePasswordCredentials( username, password ) );
+ AuthCache authCache = new BasicAuthCache();
+ BasicScheme authScheme = new BasicScheme();
+ authCache.put( host, authScheme );
+ context = new BasicHttpContext();
+ context.setAttribute( ClientContext.AUTH_CACHE, authCache );
+ } catch( GeneralSecurityException e ) {
+ throw new HadoopException( "Failed to create HTTP client.", e );
+ }
+ }
+
+ private static DefaultHttpClient createClient() throws GeneralSecurityException {
+ SchemeRegistry registry = new SchemeRegistry();
+ SSLSocketFactory socketFactory = new SSLSocketFactory( new TrustSelfSignedStrategy(), SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER );
+ registry.register( new Scheme( "https", 443, socketFactory ) );
+ registry.register( new Scheme( "http", 80, new PlainSocketFactory() ) );
+ PoolingClientConnectionManager mgr = new PoolingClientConnectionManager( registry );
+ DefaultHttpClient client = new DefaultHttpClient( mgr, new DefaultHttpClient().getParams() );
+ return client;
+ }
+
+ public String base() {
+ return base;
+ }
+
+ public HttpResponse execute( HttpRequest request ) throws IOException {
+ return client.execute( host, request, context );
+ }
+
+// SSLContext ctx = SSLContext.getInstance( "TLS" );
+// KeyManager[] keyManagers = createKeyManagers( "jks", "target/test-classes/client-keystore.jks", "horton" );
+// TrustManager[] trustManagers = createTrustManagers( "jks", "target/test-classes/client-truststore.jks", "horton" );
+// ctx.init( keyManagers, trustManagers, new SecureRandom() );
+//
+// SSLSocketFactory socketFactory = new SSLSocketFactory( ctx, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER );
+//
+// SchemeRegistry schemes = new SchemeRegistry();
+// schemes.register( new Scheme( "https", port, socketFactory ) );
+// ClientConnectionManager cm = new BasicClientConnectionManager( schemes );
+//
+// HttpClient client = new DefaultHttpClient( cm );
+//
+// HttpGet get = new HttpGet( url );
+// ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+// client.execute( get ).getEntity().writeTo( buffer );
+// assertThat( buffer.toString(), equalTo( "<html>Hello!</html>" ) );
+//}
+//
+// private static KeyManager[] createKeyManagers( String keyStoreType, String keyStorePath, String keyStorePassword ) throws Exception {
+// KeyStore keyStore = loadKeyStore( keyStoreType, keyStorePath, keyStorePassword );
+// KeyManagerFactory kmf = KeyManagerFactory.getInstance( KeyManagerFactory.getDefaultAlgorithm() );
+// kmf.init( keyStore, keyStorePassword.toCharArray() );
+// return kmf.getKeyManagers();
+// }
+//
+// private static TrustManager[] createTrustManagers( String trustStoreType, String trustStorePath, String trustStorePassword ) throws Exception {
+// KeyStore trustStore = loadKeyStore( trustStoreType, trustStorePath, trustStorePassword );
+// TrustManagerFactory tmf = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() );
+// tmf.init( trustStore );
+// return tmf.getTrustManagers();
+// }
+//
+// private static KeyStore loadKeyStore( String type, String path, String password ) throws IOException, NoSuchAlgorithmException, CertificateException, KeyStoreException {
+// KeyStore keyStore = KeyStore.getInstance( type );
+// InputStream keystoreInput = new FileInputStream( path );
+// keyStore.load( keystoreInput, password.toCharArray() );
+// return keyStore;
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/HadoopException.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/HadoopException.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/HadoopException.java
new file mode 100644
index 0000000..ebd1cb4
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/HadoopException.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell;
+
+public class HadoopException extends RuntimeException {
+
+ public HadoopException( String message ) {
+ super( message );
+ }
+
+ public HadoopException( String message, Throwable throwable ) {
+ super( message, throwable );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/KnoxShell.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/KnoxShell.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/KnoxShell.java
deleted file mode 100644
index 2ade99a..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/KnoxShell.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell;
-
-import org.codehaus.groovy.tools.shell.AnsiDetector;
-import org.codehaus.groovy.tools.shell.Groovysh;
-import org.fusesource.jansi.Ansi;
-import org.fusesource.jansi.AnsiConsole;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-
-public class KnoxShell {
-
- static {
- AnsiConsole.systemInstall();
- Ansi.setDetector( new AnsiDetector() );
- System.setProperty( "groovysh.prompt", "knox" );
- }
-
- public static void main( String... args ) {
- StringWriter buffer = new StringWriter();
- PrintWriter imports = new PrintWriter( buffer );
- imports.println( "import org.apache.hadoop.gateway.shell.hadoop.Hadoop;" );
- imports.println( "import org.apache.hadoop.gateway.shell.hdfs.Hdfs;" );
- imports.println( "import org.apache.hadoop.gateway.shell.job.Job;" );
- imports.println( "import org.apache.hadoop.gateway.shell.workflow.Workflow;" );
- Groovysh shell = new Groovysh();
- shell.execute( buffer.toString() );
- shell.run( args );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Shell.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Shell.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Shell.java
new file mode 100644
index 0000000..937c13b
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/Shell.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell;
+
+import org.codehaus.groovy.tools.shell.AnsiDetector;
+import org.codehaus.groovy.tools.shell.Groovysh;
+import org.fusesource.jansi.Ansi;
+import org.fusesource.jansi.AnsiConsole;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+public class Shell {
+
+ static {
+ AnsiConsole.systemInstall();
+ Ansi.setDetector( new AnsiDetector() );
+ System.setProperty( "groovysh.prompt", "knox" );
+ }
+
+ public static void main( String... args ) {
+ StringWriter buffer = new StringWriter();
+ PrintWriter imports = new PrintWriter( buffer );
+ imports.println( "import org.apache.hadoop.gateway.shell.Hadoop;" );
+ imports.println( "import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs;" );
+ imports.println( "import org.apache.hadoop.gateway.shell.job.Job as job;" );
+ imports.println( "import org.apache.hadoop.gateway.shell.workflow.Workflow as workflow;" );
+ Groovysh shell = new Groovysh();
+ shell.execute( buffer.toString() );
+ shell.run( args );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hadoop/Hadoop.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hadoop/Hadoop.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hadoop/Hadoop.java
deleted file mode 100644
index 9a994d8..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hadoop/Hadoop.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hadoop;
-
-import com.jayway.restassured.RestAssured;
-import com.jayway.restassured.specification.RequestSpecification;
-import com.jayway.restassured.specification.ResponseSpecification;
-
-import static com.jayway.restassured.RestAssured.expect;
-import static com.jayway.restassured.RestAssured.with;
-
-public class Hadoop {
-
- String username;
- String password;
-
- public static Hadoop login( String baseUri, String username, String password ) {
- RestAssured.baseURI = baseUri;
- return new Hadoop( username, password );
- }
-
- public RequestSpecification request() {
- //return with().log().all().auth().preemptive().basic( username, password );
- return with().auth().preemptive().basic( username, password );
- }
-
- public ResponseSpecification response() {
- return expect(); //.log().all();
- }
-
- private Hadoop( String username, String password ) {
- this.username = username;
- this.password = password;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Example.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Example.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Example.groovy
new file mode 100644
index 0000000..f6ec4c4
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Example.groovy
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs
+
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sample"
+username = "mapred"
+password = "mapred-password"
+inputFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/LICENSE"
+jarFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/hadoop-examples.jar"
+
+hadoop = Hadoop.login( gateway, username, password )
+
+println Hdfs.ls(hadoop).dir( "/" ).now().string
+
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+
+hdfs.mkdir(hadoop).dir( "/tmp/test").now()
+
+hdfs.put(hadoop).file( inputFile ).to( "/tmp/test/input/LICENSE" ).now()
+
+hdfs.get(hadoop).file( "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/OUTPUT" ).from( "/tmp/test/input/LICENSE" ).now()
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Get.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Get.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Get.java
new file mode 100644
index 0000000..1413ebd
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Get.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.hadoop.gateway.shell.HadoopException;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+public class Get {
+
+ static class Request extends AbstractRequest {
+
+ String from;
+ String to;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request from( String file ) {
+ this.from = file;
+ return this;
+ }
+
+ public Request file( String file ) {
+ this.to = file;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Hdfs.SERVICE_PATH, from );
+ addQueryParam( uri, "op", "OPEN" );
+ HttpGet request = new HttpGet( uri.build() );
+ return new Response( execute( request ), to );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response, String to ) throws IOException {
+ super( response );
+ if( to != null ) {
+ FileUtils.copyInputStreamToFile( getStream(), new File( to ) );
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Hdfs.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Hdfs.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Hdfs.java
index 2b80ed6..958e5b3 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Hdfs.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Hdfs.java
@@ -17,26 +17,30 @@
*/
package org.apache.hadoop.gateway.shell.hdfs;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
+import org.apache.hadoop.gateway.shell.Hadoop;
public class Hdfs {
static String SERVICE_PATH = "/namenode/api/v1";
- public static HdfsLsRequest ls( Hadoop hadoop ) {
- return new HdfsLsRequest( hadoop );
+ public static Ls.Request ls( Hadoop hadoop ) {
+ return new Ls.Request( hadoop );
}
- public static HdfsRmRequest rm( Hadoop hadoop ) {
- return new HdfsRmRequest( hadoop );
+ public static Rm.Request rm( Hadoop hadoop ) {
+ return new Rm.Request( hadoop );
}
- public static HdfsPutRequest put( Hadoop hadoop ) {
- return new HdfsPutRequest( hadoop );
+ public static Put.Request put( Hadoop hadoop ) {
+ return new Put.Request( hadoop );
}
- public static HdfsMkdirRequest mkdir( Hadoop hadoop ) {
- return new HdfsMkdirRequest( hadoop );
+ public static Get.Request get( Hadoop hadoop ) {
+ return new Get.Request( hadoop );
+ }
+
+ public static Mkdir.Request mkdir( Hadoop hadoop ) {
+ return new Mkdir.Request( hadoop );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsRequest.java
deleted file mode 100644
index afc1051..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsRequest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-public class HdfsLsRequest extends AbstractRequest {
-
- String dir;
-
- HdfsLsRequest( Hadoop hadoop ) {
- super( hadoop );
- request().queryParam( "op", "LISTSTATUS" );
- }
-
- public HdfsLsRequest dir( String dir ) {
- this.dir = dir;
- return this;
- }
-
- public HdfsLsResponse go() {
- return new HdfsLsResponse( request().get( Hdfs.SERVICE_PATH + dir ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsResponse.java
deleted file mode 100644
index eac22d9..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsLsResponse.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-public class HdfsLsResponse extends AbstractResponse {
-
- HdfsLsResponse( Response response ) {
- super( response );
- }
-
- public String getJson() {
- return response().asString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirRequest.java
deleted file mode 100644
index 2267bda..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirRequest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-public class HdfsMkdirRequest extends AbstractRequest {
-
- String dir;
-
- HdfsMkdirRequest( Hadoop hadoop ) {
- super( hadoop );
- request().queryParam( "op", "MKDIRS" );
- }
-
- public HdfsMkdirRequest perm( String perm ) {
- request().queryParam( "permission", perm );
- return this;
- }
-
- public HdfsMkdirRequest dir( String dir ) {
- this.dir = dir;
- return this;
- }
-
- public HdfsMkdirResponse go() {
- return new HdfsMkdirResponse( request().put( Hdfs.SERVICE_PATH + dir ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirResponse.java
deleted file mode 100644
index ceb6dbe..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsMkdirResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-public class HdfsMkdirResponse extends AbstractResponse {
-
- HdfsMkdirResponse( Response response ) {
- super( response );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutRequest.java
deleted file mode 100644
index fb1238d..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutRequest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import com.jayway.restassured.response.Response;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-import java.io.File;
-import java.io.IOException;
-
-import static com.jayway.restassured.RestAssured.with;
-
-public class HdfsPutRequest extends AbstractRequest {
-
- String from;
- String to;
-
- HdfsPutRequest( Hadoop hadoop ) {
- super( hadoop );
- }
-
- public HdfsPutRequest from( String file ) {
- this.from = file;
- return this;
- }
-
- public HdfsPutRequest to( String file ) {
- this.to = file;
- return this;
- }
-
- public HdfsPutResponse go() throws IOException {
- Response response = with().spec( hadoop().request() ).queryParam( "op", "CREATE" )
- .expect().statusCode( 307 ).when().put( Hdfs.SERVICE_PATH + to );
- String url = response.getHeader( "Location" );
- request().body( FileUtils.readFileToByteArray( new File( from ) ) );
- return new HdfsPutResponse( request().put( url ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutResponse.java
deleted file mode 100644
index 828d2b8..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsPutResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-public class HdfsPutResponse extends AbstractResponse {
-
- HdfsPutResponse( Response response ) {
- super( response );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmRequest.java
deleted file mode 100644
index 8d88bea..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmRequest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-public class HdfsRmRequest extends AbstractRequest {
-
- String file;
-
- HdfsRmRequest( Hadoop hadoop ) {
- super( hadoop );
- request().queryParam( "op", "DELETE" );
- }
-
- public HdfsRmRequest file( String file ) {
- this.file = file;
- return this;
- }
-
- public HdfsRmRequest recursive( boolean recursive ) {
- request().queryParam( "recursive", Boolean.toString( recursive ) );
- return this;
- }
-
- public HdfsRmRequest recursive() {
- return recursive( true );
- }
-
- public HdfsRmResponse go() {
- return new HdfsRmResponse( request().delete( Hdfs.SERVICE_PATH + file ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmResponse.java
deleted file mode 100644
index e80f8b0..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/HdfsRmResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.hdfs;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-public class HdfsRmResponse extends AbstractResponse {
-
- HdfsRmResponse( Response response ) {
- super( response );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Ls.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Ls.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Ls.java
new file mode 100644
index 0000000..25688ac
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Ls.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Ls {
+
+ static class Request extends AbstractRequest {
+
+ String dir;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request dir( String dir ) {
+ this.dir = dir;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Hdfs.SERVICE_PATH, dir );
+ addQueryParam( uri, "op", "LISTSTATUS" );
+ HttpGet get = new HttpGet( uri.build() );
+ return new Response( execute( get ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Mkdir.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Mkdir.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Mkdir.java
new file mode 100644
index 0000000..2212d9b
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Mkdir.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Mkdir {
+
+ static class Request extends AbstractRequest {
+
+ String dir = null;
+ String perm = null;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request perm( String perm ) {
+ this.perm = perm;
+ return this;
+ }
+
+ public Request dir( String dir ) {
+ this.dir = dir;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Hdfs.SERVICE_PATH, dir );
+ addQueryParam( uri, "op", "MKDIRS" );
+ addQueryParam( uri, "permissions", perm );
+ HttpPut request = new HttpPut( uri.build() );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ consume();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Put.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Put.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Put.java
new file mode 100644
index 0000000..08fc877
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Put.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.hadoop.gateway.shell.HadoopException;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Put {
+
+ static class Request extends AbstractRequest {
+
+ String text;
+ String from;
+ String to;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request text( String text ) {
+ this.text = text;
+ return this;
+ }
+
+ public Request file( String file ) {
+ this.from = file;
+ return this;
+ }
+
+ public Request to( String file ) {
+ this.to = file;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Hdfs.SERVICE_PATH, to );
+ addQueryParam( uri, "op", "CREATE" );
+ HttpPut nn = new HttpPut( uri.build() );
+ HttpResponse r = execute( nn );
+ if( r.getStatusLine().getStatusCode() != HttpStatus.SC_TEMPORARY_REDIRECT ) {
+ throw new HadoopException( r.getStatusLine().toString() );
+ }
+ EntityUtils.consumeQuietly( r.getEntity() );
+ Header[] h = r.getHeaders( "Location" );
+ if( h == null || h.length != 1 ) {
+ throw new HadoopException( "Invalid Location header." );
+ }
+ String loc = h[0].getValue();
+ HttpPut dn = new HttpPut( loc );
+ HttpEntity e = null;
+ if( text != null ) {
+ e = new StringEntity( text );
+ } else if( from != null ) {
+ e = new FileEntity( new File( from ) );
+ }
+ dn.setEntity( e );
+ return new Response( execute( dn ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ consume();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Rm.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Rm.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Rm.java
new file mode 100644
index 0000000..fb2b2f8
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/hdfs/Rm.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.hdfs;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Rm {
+
+ static class Request extends AbstractRequest {
+
+ String file;
+ Boolean recursive;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request file( String file ) {
+ this.file = file;
+ return this;
+ }
+
+ public Request recursive( Boolean recursive ) {
+ this.recursive = recursive;
+ return this;
+ }
+
+ public Request recursive() {
+ return recursive( true );
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Hdfs.SERVICE_PATH, file );
+ addQueryParam( uri, "op", "DELETE" );
+ addQueryParam( uri, "recursive", recursive );
+ HttpDelete request = new HttpDelete( uri.build() );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) throws IOException {
+ super( response );
+ consume();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleHive.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleHive.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleHive.groovy
new file mode 100644
index 0000000..8f09906
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleHive.groovy
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job
+
+import org.apache.hadoop.gateway.shell.job.Job as job
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+
+import com.jayway.jsonpath.JsonPath
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sample"
+username = "mapred"
+password = "mapred-password"
+inputFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/LICENSE"
+
+script = "select+*+from+pokes;"
+
+hadoop = Hadoop.login( gateway, username, password )
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+hdfs.put(hadoop).text( script ).to( "/tmp/test/script.hive" ).now()
+hdfs.put(hadoop).file( inputFile ).to( "/tmp/test/input/FILE" ).now()
+
+jobId = job.submitHive(hadoop)
+ .file( "/tmp/test/script.hive" ) \
+ .arg( "-v" ) \
+ .statusDir( "/tmp/test/output" ) \
+ .now().jobId
+println "Job=" + jobId
+
+println job.queryQueue(hadoop).now().string
+
+done = false;
+while( !done ) {
+ json = job.queryStatus(hadoop).jobId(jobId).now().string
+ done = JsonPath.read( json, "\$.status.jobComplete" )
+ sleep( 1000 )
+ print "."
+}
+println "done"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleJava.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleJava.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleJava.groovy
new file mode 100644
index 0000000..6681fa1
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExampleJava.groovy
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job
+
+import org.apache.hadoop.gateway.shell.job.Job as job
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+
+import com.jayway.jsonpath.JsonPath
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sample"
+username = "mapred"
+password = "mapred-password"
+inputFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/LICENSE"
+jarFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/hadoop-examples.jar"
+
+hadoop = Hadoop.login( gateway, username, password )
+
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+hdfs.mkdir(hadoop).dir( "/tmp/test").now()
+hdfs.put(hadoop).file( inputFile ).to( "/tmp/test/input/FILE" ).now()
+hdfs.put(hadoop).file( jarFile ).to( "/tmp/test/hadoop-examples.jar" ).now()
+
+jobId = job.submitJava(hadoop) \
+ .jar( "/tmp/test/hadoop-examples.jar" ) \
+ .app( "wordcount" ) \
+ .input( "/tmp/test/input" ) \
+ .output( "/tmp/test/output" ) \
+ .now().jobId
+println "Job=" + jobId
+
+println job.queryQueue(hadoop).now().string
+
+done = false;
+while( !done ) {
+ json = job.queryStatus(hadoop).jobId(jobId).now().string
+ done = JsonPath.read( json, "\$.status.jobComplete" )
+ sleep( 1000 )
+ print "."
+}
+println "done"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExamplePig.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExamplePig.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExamplePig.groovy
new file mode 100644
index 0000000..734208a
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/ExamplePig.groovy
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job
+
+import org.apache.hadoop.gateway.shell.job.Job as job
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+
+import com.jayway.jsonpath.JsonPath
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sample"
+username = "mapred"
+password = "mapred-password"
+inputFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/LICENSE"
+
+script = """\
+A = load '/tmp/test/input/FILE' using PigStorage(':');
+B = foreach A generate \$0 as id;
+dump B;
+"""
+
+hadoop = Hadoop.login( gateway, username, password )
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+hdfs.put(hadoop).file( inputFile ).to( "/tmp/test/input/FILE" ).now()
+hdfs.put(hadoop).text( script ).to( "/tmp/test/script.pig" ).now()
+
+jobId = job.submitPig(hadoop)
+ .file( "/tmp/test/script.pig" ) \
+ .arg( "-v" ) \
+ .statusDir( "/tmp/test/output" ) \
+ .now().jobId
+println "Job=" + jobId
+
+println job.queryQueue(hadoop).now().string
+
+done = false;
+while( !done ) {
+ json = job.queryStatus(hadoop).jobId(jobId).now().string
+ done = JsonPath.read( json, "\$.status.jobComplete" )
+ sleep( 1000 )
+ print "."
+}
+println "done"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Hive.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Hive.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Hive.java
new file mode 100644
index 0000000..2a8a410
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Hive.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Hive {
+
+ static class Request extends AbstractRequest {
+
+ private String group;
+ private String file;
+ private String statusDir;
+ List<NameValuePair> params = new ArrayList<NameValuePair>();
+
+ public Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request group( String group ) {
+ this.group = group;
+ return this;
+ }
+
+ public Request file( String file ) {
+ this.file = file;
+ return this;
+ }
+
+ public Request statusDir( String dir ) {
+ this.statusDir = dir;
+ return this;
+ }
+
+ public Request arg( String value ) {
+ addParam( params, "arg", value );
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Job.SERVICE_PATH, "/hive" );
+ addParam( params, "group", group );
+ addParam( params, "file", file );
+ addParam( params, "statusdir", statusDir );
+ UrlEncodedFormEntity form = new UrlEncodedFormEntity( params );
+ HttpPost request = new HttpPost( uri.build() );
+ request.setEntity( form );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ protected Response( HttpResponse response ) {
+ super( response );
+ }
+
+ public String getJobId() throws IOException {
+ return JsonPath.read( getString(), "$.id" );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
new file mode 100644
index 0000000..dc416ed
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.message.BasicNameValuePair;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+class Java {
+
+ static class Request extends AbstractRequest {
+
+ String jar;
+ String app;
+ String input;
+ String output;
+
+ public Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request jar( String jar ) {
+ this.jar = jar;
+ return this;
+ }
+
+ public Request app( String app ) {
+ this.app = app;
+ return this;
+ }
+
+ public Request input( String dir ) {
+ input = dir;
+ return this;
+ }
+
+ public Request output( String dir ) {
+ output = dir;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Job.SERVICE_PATH, "/mapreduce/jar" );
+ List<NameValuePair> params = new ArrayList<NameValuePair>();
+ params.add( new BasicNameValuePair( "jar", jar ) );
+ params.add( new BasicNameValuePair( "class", app ) );
+ params.add( new BasicNameValuePair( "arg", input ) );
+ params.add( new BasicNameValuePair( "arg", output ) );
+ UrlEncodedFormEntity form = new UrlEncodedFormEntity( params );
+ HttpPost request = new HttpPost( uri.build() );
+ request.setEntity( form );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ public Response( HttpResponse response ) {
+ super( response );
+ }
+
+ public String getJobId() throws IOException {
+ return JsonPath.read( getString(), "$.id" );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
index 0684a43..1980a06 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java
@@ -17,14 +17,30 @@
*/
package org.apache.hadoop.gateway.shell.job;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
+import org.apache.hadoop.gateway.shell.Hadoop;
public class Job {
static String SERVICE_PATH = "/templeton/api/v1";
- public static JobJavaRequest java( Hadoop hadoop ) {
- return new JobJavaRequest( hadoop );
+ public static Java.Request submitJava( Hadoop hadoop ) {
+ return new Java.Request( hadoop );
+ }
+
+ public static Pig.Request submitPig( Hadoop hadoop ) {
+ return new Pig.Request( hadoop );
+ }
+
+ public static Hive.Request submitHive( Hadoop hadoop ) {
+ return new Hive.Request( hadoop );
+ }
+
+ public static Queue.Request queryQueue( Hadoop hadoop ) {
+ return new Queue.Request( hadoop );
+ }
+
+ public static Status.Request queryStatus( Hadoop hadoop ) {
+ return new Status.Request( hadoop );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaRequest.java
deleted file mode 100644
index 14a8838..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaRequest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.job;
-
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-public class JobJavaRequest extends AbstractRequest {
-
- String input;
- String output;
-
- public JobJavaRequest( Hadoop hadoop ) {
- super( hadoop );
- }
-
- public JobJavaRequest jar( String jar ) {
- request().formParam( "jar", jar );
- return this;
- }
-
- public JobJavaRequest app( String app ) {
- request().formParam( "class", app );
- return this;
- }
-
- public JobJavaRequest input( String dir ) {
- input = dir;
- return this;
- }
-
- public JobJavaRequest output( String dir ) {
- output = dir;
- return this;
- }
-
- public JobJavaResponse go() {
- request().formParam( "arg", input );
- request().formParam( "arg", output );
- return new JobJavaResponse( request().post( Job.SERVICE_PATH + "/mapreduce/jar" ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaResponse.java
deleted file mode 100644
index 65a3166..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/JobJavaResponse.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.job;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-import static com.jayway.restassured.path.json.JsonPath.from;
-
-public class JobJavaResponse extends AbstractResponse {
-
- String json = null;
-
- public JobJavaResponse( Response response ) {
- super( response );
- }
-
- public String getJson() {
- if( json == null ) {
- json = response().asString();
- }
- return json;
- }
-
- public String getJobId() {
- return from( getJson() ).getString( "id" );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
new file mode 100644
index 0000000..afa277d
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class Pig {
+
+ static class Request extends AbstractRequest {
+
+ private String group;
+ private String file;
+ private String arg;
+ private String statusDir;
+
+ public Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request group( String group ) {
+ this.group = group;
+ return this;
+ }
+
+ public Request file( String file ) {
+ this.file = file;
+ return this;
+ }
+
+ public Request arg( String arg ) {
+ this.arg = arg;
+ return this;
+ }
+
+ public Request statusDir( String dir ) {
+ this.statusDir = dir;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Job.SERVICE_PATH, "/pig" );
+ List<NameValuePair> params = new ArrayList<NameValuePair>();
+ addParam( params, "group", group );
+ addParam( params, "file", file );
+ addParam( params, "arg", arg );
+ addParam( params, "statusdir", statusDir );
+ UrlEncodedFormEntity form = new UrlEncodedFormEntity( params );
+ HttpPost request = new HttpPost( uri.build() );
+ request.setEntity( form );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ protected Response( HttpResponse response ) {
+ super( response );
+ }
+
+ public String getJobId() throws IOException {
+ return JsonPath.read( getString(), "$.id" );
+ }
+
+ }
+
+}