You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by lm...@apache.org on 2016/07/14 21:13:39 UTC
[1/2] knox git commit: KNOX-715 - submitJava should allow multiple
argument via arg() Pierre Regazzoni via lmccay
Repository: knox
Updated Branches:
refs/heads/v0.9.0 d1e3eab85 -> 980d37026
KNOX-715 - submitJava should allow multiple argument via arg() Pierre Regazzoni via lmccay
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/58da90db
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/58da90db
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/58da90db
Branch: refs/heads/v0.9.0
Commit: 58da90db46e97b9f78bae8810c6a2fc52c603258
Parents: d1e3eab
Author: Larry McCay <lm...@hortonworks.com>
Authored: Sat Jul 2 00:14:57 2016 -0400
Committer: Larry McCay <lm...@hortonworks.com>
Committed: Thu Jul 14 17:12:16 2016 -0400
----------------------------------------------------------------------
.../samples/ExampleWebHCatJobTeragen.groovy | 80 ++++++++++++++++++++
.../apache/hadoop/gateway/shell/job/Java.java | 7 +-
2 files changed, 86 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/58da90db/gateway-release/home/samples/ExampleWebHCatJobTeragen.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatJobTeragen.groovy b/gateway-release/home/samples/ExampleWebHCatJobTeragen.groovy
new file mode 100644
index 0000000..3b111a9
--- /dev/null
+++ b/gateway-release/home/samples/ExampleWebHCatJobTeragen.groovy
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import com.jayway.jsonpath.JsonPath
+import groovy.json.JsonSlurper
+import org.apache.hadoop.gateway.shell.Hadoop
+import org.apache.hadoop.gateway.shell.hdfs.Hdfs
+import org.apache.hadoop.gateway.shell.job.Job
+
+import static java.util.concurrent.TimeUnit.SECONDS
+import org.apache.hadoop.gateway.shell.Credentials
+
+gateway = "https://localhost:8443/gateway/sandbox"
+
+// You will need to copy hadoop-mapreduce-samples.jar from your cluster
+// and place it under samples/ directory.
+// For example you might find the jar under: /usr/iop/current/hadoop-mapreduce-client
+jarFile = "samples/hadoop-mapreduce-examples.jar"
+
+credentials = new Credentials()
+credentials.add("ClearInput", "Enter username: ", "user")
+ .add("HiddenInput", "Enter pas" + "sword: ", "pass")
+credentials.collect()
+
+username = credentials.get("user").string()
+pass = credentials.get("pass").string()
+
+jobDir = "/user/" + username + "/test"
+
+session = Hadoop.login( gateway, username, pass )
+
+println "Delete " + jobDir + ": " + Hdfs.rm( session ).file( jobDir ).recursive().now().statusCode
+println "Create " + jobDir + ": " + Hdfs.mkdir( session ).dir( jobDir ).now().statusCode
+
+putJar = Hdfs.put( session ).file( jarFile ).to( jobDir + "/lib/hadoop-mapreduce-examples.jar" ).later() {
+ println "Put " + jobDir + "/lib/hadoop-mapreduce-examples.jar: " + it.statusCode }
+
+session.waitFor( putJar )
+
+// Run teragen with 5 mappers. It will generate 500 records of 100 bytes each.
+jobId = Job.submitJava(session) \
+ .jar( jobDir + "/lib/hadoop-mapreduce-examples.jar" ) \
+ .app( "teragen" ) \
+ .arg( "-D").arg("mapred.map.tasks=5") \
+ .arg( "500" ) \
+ .input( jobDir + "/input_terasort" ) \
+ .now().jobId
+println "Submitted job: " + jobId
+
+println "Polling up to 60s for job completion..."
+done = false
+count = 0
+while( !done && count++ < 90 ) {
+ sleep( 1000 )
+ json = Job.queryStatus(session).jobId(jobId).now().string
+ done = JsonPath.read( json, "\$.status.jobComplete" )
+ print "."; System.out.flush();
+}
+println ""
+println "Job status: " + done
+
+text = Hdfs.ls( session ).dir( jobDir + "/input_terasort" ).now().string
+json = (new JsonSlurper()).parseText( text )
+println json.FileStatuses.FileStatus.pathSuffix
+
+println "Session closed: " + session.shutdown( 10, SECONDS )
http://git-wip-us.apache.org/repos/asf/knox/blob/58da90db/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
----------------------------------------------------------------------
diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
index da348ba..93d3706 100644
--- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
+++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Java.java
@@ -41,6 +41,7 @@ class Java {
String app;
String input;
String output;
+ List<NameValuePair> params = new ArrayList<NameValuePair>();
public Request( Hadoop session ) {
super( session );
@@ -66,12 +67,16 @@ class Java {
return this;
}
+ public Request arg( String value ) {
+ addParam( params, "arg", value );
+ return this;
+ }
+
protected Callable<Response> callable() {
return new Callable<Response>() {
@Override
public Response call() throws Exception {
URIBuilder uri = uri( Job.SERVICE_PATH, "/mapreduce/jar" );
- List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add( new BasicNameValuePair( "jar", jar ) );
params.add( new BasicNameValuePair( "class", app ) );
params.add( new BasicNameValuePair( "arg", input ) );
[2/2] knox git commit: KNOX-722 - Null Pointer Exception while Knox
SSO configured for OKTA/SAML v2
Posted by lm...@apache.org.
KNOX-722 - Null Pointer Exception while Knox SSO configured for OKTA/SAML v2
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/980d3702
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/980d3702
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/980d3702
Branch: refs/heads/v0.9.0
Commit: 980d37026d8a12ad7c3f8cbc8aec702a347ac867
Parents: 58da90d
Author: Larry McCay <lm...@hortonworks.com>
Authored: Thu Jul 14 17:11:05 2016 -0400
Committer: Larry McCay <lm...@hortonworks.com>
Committed: Thu Jul 14 17:13:07 2016 -0400
----------------------------------------------------------------------
gateway-provider-security-pac4j/pom.xml | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/980d3702/gateway-provider-security-pac4j/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-provider-security-pac4j/pom.xml b/gateway-provider-security-pac4j/pom.xml
index 89a0ab4..5ff4e41 100644
--- a/gateway-provider-security-pac4j/pom.xml
+++ b/gateway-provider-security-pac4j/pom.xml
@@ -36,8 +36,8 @@
</licenses>
<properties>
- <j2e-pac4j.version>1.2.1</j2e-pac4j.version>
- <pac4j.version>1.8.5</pac4j.version>
+ <j2e-pac4j.version>1.2.2</j2e-pac4j.version>
+ <pac4j.version>1.8.9</pac4j.version>
</properties>
<dependencies>
@@ -51,6 +51,12 @@
<groupId>org.pac4j</groupId>
<artifactId>j2e-pac4j</artifactId>
<version>${j2e-pac4j.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.pac4j</groupId>
+ <artifactId>pac4j-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -61,6 +67,12 @@
<dependency>
<groupId>org.pac4j</groupId>
+ <artifactId>pac4j-core</artifactId>
+ <version>${pac4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.pac4j</groupId>
<artifactId>pac4j-config</artifactId>
<version>${pac4j.version}</version>
<exclusions>