You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2013/03/15 18:38:50 UTC
[2/3] Client groovy examples all working. Including SSL.
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Queue.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Queue.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Queue.java
new file mode 100644
index 0000000..874694e
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Queue.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Queue {
+
+ static class Request extends AbstractRequest {
+
+ public Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Job.SERVICE_PATH, "/queue" );
+ HttpGet request = new HttpGet( uri.build() );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ protected Response( HttpResponse response ) {
+ super( response );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Status.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Status.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Status.java
new file mode 100644
index 0000000..c3fdddf
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Status.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.job;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Status {
+
+ static class Request extends AbstractRequest {
+
+ private String jobId;
+
+ public Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request jobId( String jobId ) {
+ this.jobId = jobId;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Job.SERVICE_PATH, "/queue/", jobId );
+ HttpGet request = new HttpGet( uri.build() );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ protected Response( HttpResponse response ) {
+ super( response );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Example.groovy
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Example.groovy b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Example.groovy
new file mode 100644
index 0000000..16a698f
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Example.groovy
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.workflow
+
+import org.apache.hadoop.gateway.shell.workflow.Workflow as workflow;
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs as hdfs
+
+import com.jayway.jsonpath.JsonPath
+import org.apache.hadoop.gateway.shell.Hadoop
+
+gateway = "https://localhost:8443/gateway/sample"
+jobTracker = "sandbox:50300";
+nameNode = "sandbox:8020";
+username = "mapred"
+password = "mapred-password"
+inputFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/LICENSE"
+jarFile = "/Users/kevin.minder/Projects/gateway-0.2.0-SNAPSHOT/hadoop-examples.jar"
+
+definition = """\
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="wordcount-workflow">
+ <start to="root-node"/>
+ <action name="root-node">
+ <java>
+ <job-tracker>$jobTracker</job-tracker>
+ <name-node>hdfs://$nameNode</name-node>
+ <main-class>org.apache.hadoop.examples.WordCount</main-class>
+ <arg>/tmp/test/input</arg>
+ <arg>/tmp/test/output</arg>
+ </java>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+ <kill name="fail">
+ <message>Java failed, error message[\${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
+"""
+
+configuration = """\
+<configuration>
+ <property>
+ <name>user.name</name>
+ <value>$username</value>
+ </property>
+ <property>
+ <name>oozie.wf.application.path</name>
+ <value>hdfs://$nameNode/tmp/test</value>
+ </property>
+</configuration>
+"""
+
+hadoop = Hadoop.login( gateway, username, password )
+
+hdfs.rm(hadoop).file( "/tmp/test" ).recursive().now()
+hdfs.mkdir(hadoop).dir( "/tmp/test").now()
+hdfs.put(hadoop).text( definition ).to( "/tmp/test/workflow.xml" ).now()
+hdfs.put(hadoop).file( inputFile ).to( "/tmp/test/input/FILE" ).now()
+hdfs.put(hadoop).file( jarFile ).to( "/tmp/test/lib/hadoop-examples.jar" ).now()
+
+jobId = workflow.submit(hadoop).text( configuration ).now().jobId
+println "Job=" + jobId
+
+status = "UNKNOWN";
+while( status != "SUCCEEDED" ) {
+ json = workflow.status(hadoop).jobId(jobId).now().string
+ status = JsonPath.read( json, "\$.status" )
+ sleep( 1000 )
+ print "."
+}
+println "done"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Status.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Status.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Status.java
new file mode 100644
index 0000000..6b9d196
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Status.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.workflow;
+
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.utils.URIBuilder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Status {
+
+ static class Request extends AbstractRequest {
+
+ private String jobId;
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request jobId( String jobId ) {
+ this.jobId = jobId;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Workflow.SERVICE_PATH, "/job/", jobId );
+ HttpGet request = new HttpGet( uri.build() );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Submit.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Submit.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Submit.java
new file mode 100644
index 0000000..ee72eae
--- /dev/null
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Submit.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.gateway.shell.workflow;
+
+import com.jayway.jsonpath.JsonPath;
+import org.apache.hadoop.gateway.shell.AbstractRequest;
+import org.apache.hadoop.gateway.shell.AbstractResponse;
+import org.apache.hadoop.gateway.shell.Hadoop;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.entity.StringEntity;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+class Submit {
+
+ static class Request extends AbstractRequest {
+
+ private String text;
+ private String file;
+ private String action = "start";
+
+ Request( Hadoop hadoop ) {
+ super( hadoop );
+ }
+
+ public Request text( String text ) {
+ this.text = text;
+ return this;
+ }
+
+ public Request file( String file ) {
+ this.file = file;
+ return this;
+ }
+
+ public Request action( String action ) {
+ this.action = action;
+ return this;
+ }
+
+ public Response now() throws IOException, URISyntaxException {
+ URIBuilder uri = uri( Workflow.SERVICE_PATH, "/jobs" );
+ addQueryParam( uri, "action", action );
+ HttpPost request = new HttpPost( uri.build() );
+ HttpEntity entity = null;
+ if( text != null ) {
+ entity = new StringEntity( text, ContentType.create( "application/xml", "UTF-8" ) );
+ } else if( file != null ) {
+ entity = new FileEntity( new File( file ), ContentType.create( "application/xml" ) );
+ }
+ request.setEntity( entity );
+ return new Response( execute( request ) );
+ }
+
+ }
+
+ static class Response extends AbstractResponse {
+
+ Response( HttpResponse response ) {
+ super( response );
+ }
+
+ public String getJobId() throws IOException {
+ return JsonPath.read( getString(), "$.id" );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Workflow.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Workflow.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Workflow.java
index 0be9304..4326957 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Workflow.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/Workflow.java
@@ -17,14 +17,18 @@
*/
package org.apache.hadoop.gateway.shell.workflow;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
+import org.apache.hadoop.gateway.shell.Hadoop;
public class Workflow {
- static String SERVICE_PATH = "oozie";
+ static String SERVICE_PATH = "/oozie/api/v1";
- public static WorkflowSubmitRequest submit( Hadoop hadoop ) {
- return new WorkflowSubmitRequest( hadoop );
+ public static Submit.Request submit( Hadoop hadoop ) {
+ return new Submit.Request( hadoop );
+ }
+
+ public static Status.Request status( Hadoop hadoop ) {
+ return new Status.Request( hadoop );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitRequest.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitRequest.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitRequest.java
deleted file mode 100644
index 62aab83..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitRequest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.workflow;
-
-import org.apache.hadoop.gateway.shell.AbstractRequest;
-import org.apache.hadoop.gateway.shell.hadoop.Hadoop;
-
-class WorkflowSubmitRequest extends AbstractRequest {
-
- WorkflowSubmitRequest( Hadoop hadoop ) {
- super( hadoop );
- }
-
- public WorkflowSubmitResponse go() {
- return new WorkflowSubmitResponse( request().post( Workflow.SERVICE_PATH ) );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitResponse.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitResponse.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitResponse.java
deleted file mode 100644
index 5aa78a9..0000000
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/workflow/WorkflowSubmitResponse.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.gateway.shell.workflow;
-
-import com.jayway.restassured.response.Response;
-import org.apache.hadoop.gateway.shell.AbstractResponse;
-
-class WorkflowSubmitResponse extends AbstractResponse {
-
- WorkflowSubmitResponse( Response response ) {
- super( response );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/37b6d21d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 93fa7c5..ed628c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -400,6 +400,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ </dependency>
+
+ <dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>