You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2013/10/01 14:31:54 UTC
git commit: KNOX-172: Partial support for ~. WebHCat still
outstanding.
Updated Branches:
refs/heads/master 832e86c86 -> 34492a258
KNOX-172: Partial support for ~. WebHCat still outstanding.
Project: http://git-wip-us.apache.org/repos/asf/incubator-knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-knox/commit/34492a25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-knox/tree/34492a25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-knox/diff/34492a25
Branch: refs/heads/master
Commit: 34492a25850271db552d84a83c61a58e90e216d6
Parents: 832e86c
Author: Kevin Minder <ke...@hortonworks.com>
Authored: Tue Oct 1 08:31:46 2013 -0400
Committer: Kevin Minder <ke...@hortonworks.com>
Committed: Tue Oct 1 08:31:46 2013 -0400
----------------------------------------------------------------------
.../rewrite/api/UrlRewriteProcessorTest.java | 31 ++++++++++++++-
.../api/UrlRewriteProcessorTest/rewrite.xml | 9 +++++
.../home/samples/ExampleOozieWorkflow.groovy | 40 ++++++++------------
.../home/samples/ExampleWebHCatJob.groovy | 2 +
gateway-release/home/templates/sandbox-1-3.xml | 5 ---
.../OozieDeploymentContributor/rewrite.xml | 19 ++++++++--
.../hadoop/gateway/security/SubjectUtils.java | 5 +--
7 files changed, 75 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java b/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
index b01b656..86d8460 100644
--- a/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
+++ b/gateway-provider-rewrite/src/test/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.gateway.filter.rewrite.api;
import org.apache.hadoop.gateway.util.urltemplate.Parser;
import org.apache.hadoop.gateway.util.urltemplate.Template;
import org.easymock.EasyMock;
-import org.junit.Ignore;
import org.junit.Test;
import javax.servlet.http.HttpServletRequest;
@@ -152,6 +151,36 @@ public class UrlRewriteProcessorTest {
}
@Test
+ public void testRewriteViaRuleWithComplexFlow() throws Exception {
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ HttpServletRequest request = EasyMock.createNiceMock( HttpServletRequest.class );
+ HttpServletResponse response = EasyMock.createNiceMock( HttpServletResponse.class );
+ EasyMock.replay( environment, request, response );
+
+ UrlRewriteProcessor processor = new UrlRewriteProcessor();
+ UrlRewriteRulesDescriptor config = UrlRewriteRulesDescriptorFactory.load(
+ "xml", getTestResourceReader( "rewrite.xml", "UTF-8" ) );
+ processor.initialize( environment, config );
+
+ Template inputUrl;
+ Template outputUrl;
+
+ inputUrl = Parser.parse( "test-scheme://test-host:777/test-path" );
+ outputUrl = processor.rewrite( null, inputUrl, UrlRewriter.Direction.IN, "test-rule-with-complex-flow" );
+ assertThat(
+ "Expect rewrite to contain the correct path.",
+ outputUrl.toString(), is( "test-scheme-output://test-host-output:42/test-path-output/test-path" ) );
+
+ inputUrl = Parser.parse( "test-scheme://test-host:42/~/test-path" );
+ outputUrl = processor.rewrite( null, inputUrl, UrlRewriter.Direction.IN, "test-rule-with-complex-flow" );
+ assertThat(
+ "Expect rewrite to contain the correct path.",
+ outputUrl.toString(), is( "test-scheme-output://test-host-output:777/test-path-output/test-home/test-path" ) );
+
+ processor.destroy();
+ }
+
+ @Test
public void testRewriteViaRuleWithWildcardTemplateAndOptionalQuery() throws Exception {
UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
HttpServletRequest request = EasyMock.createNiceMock( HttpServletRequest.class );
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml b/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
index 998f54c..84836d5 100644
--- a/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
+++ b/gateway-provider-rewrite/src/test/resources/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessorTest/rewrite.xml
@@ -24,4 +24,13 @@
<rewrite template="test-scheme-output://test-host-output:777/test-path-output/{path}?{**}"/>
</rule>
+ <rule name="test-rule-with-complex-flow" flow="OR">
+ <match pattern="*://*:*/~/{path=**}?{**}">
+ <rewrite template="test-scheme-output://test-host-output:777/test-path-output/test-home/{path}?{**}"/>
+ </match>
+ <match pattern="*://*:*/{path=**}?{**}">
+ <rewrite template="test-scheme-output://test-host-output:42/test-path-output/{path}?{**}"/>
+ </match>
+ </rule>
+
</rules>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/samples/ExampleOozieWorkflow.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleOozieWorkflow.groovy b/gateway-release/home/samples/ExampleOozieWorkflow.groovy
index 3f5c0ed..157e4ac 100644
--- a/gateway-release/home/samples/ExampleOozieWorkflow.groovy
+++ b/gateway-release/home/samples/ExampleOozieWorkflow.groovy
@@ -38,8 +38,8 @@ definition = """\
<job-tracker>\${jobTracker}</job-tracker>
<name-node>\${nameNode}</name-node>
<main-class>org.apache.hadoop.examples.WordCount</main-class>
- <arg>$jobDir/input</arg>
- <arg>$jobDir/output</arg>
+ <arg>\${inputDir}</arg>
+ <arg>\${outputDir}</arg>
</java>
<ok to="end"/>
<error to="fail"/>
@@ -54,7 +54,7 @@ definition = """\
configuration = """\
<configuration>
<property>
- <name>fs.default.name</name>
+ <name>user.name</name>
<value>default</value>
</property>
<property>
@@ -62,33 +62,21 @@ configuration = """\
<value>default</value>
</property>
<property>
- <name>mapred.job.tracker</name>
- <value>default</value>
- </property>
- <property>
<name>jobTracker</name>
<value>default</value>
</property>
<property>
- <name>user.name</name>
- <value>default</value>
+ <name>inputDir</name>
+ <value>$jobDir/input</value>
</property>
<property>
- <name>mapreduce.job.user.name</name>
- <value>default</value>
+ <name>outputDir</name>
+ <value>$jobDir/output</value>
</property>
<property>
<name>oozie.wf.application.path</name>
<value>$jobDir</value>
</property>
- <property>
- <name>oozie.libpath</name>
- <value>$jobDir/lib</value>
- </property>
- <property>
- <name>oozie.proxysubmission</name>
- <value>true</value>
- </property>
</configuration>
"""
@@ -112,17 +100,21 @@ jobId = Workflow.submit(session).text( configuration ).now().jobId
println "Submitted job: " + jobId
println "Polling up to 60s for job completion..."
-status = "UNKNOWN";
+status = "RUNNING";
count = 0;
-while( status != "SUCCEEDED" && count++ < 60 ) {
+while( status == "RUNNING" && count++ < 60 ) {
sleep( 1000 )
json = Workflow.status(session).jobId( jobId ).now().string
status = JsonPath.read( json, "\$.status" )
+ print "."; System.out.flush();
}
+println ""
println "Job status: " + status
-text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
-json = (new JsonSlurper()).parseText( text )
-println json.FileStatuses.FileStatus.pathSuffix
+if( status == "SUCCEEDED" ) {
+ text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
+ json = (new JsonSlurper()).parseText( text )
+ println json.FileStatuses.FileStatus.pathSuffix
+}
println "Session closed: " + session.shutdown( 10, SECONDS )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/samples/ExampleWebHCatJob.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatJob.groovy b/gateway-release/home/samples/ExampleWebHCatJob.groovy
index d0432ba..50e43fd 100644
--- a/gateway-release/home/samples/ExampleWebHCatJob.groovy
+++ b/gateway-release/home/samples/ExampleWebHCatJob.groovy
@@ -58,7 +58,9 @@ while( !done && count++ < 60 ) {
sleep( 1000 )
json = Job.queryStatus(session).jobId(jobId).now().string
done = JsonPath.read( json, "\$.status.jobComplete" )
+ print "."; System.out.flush();
}
+println ""
println "Job status: " + done
text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-release/home/templates/sandbox-1-3.xml
----------------------------------------------------------------------
diff --git a/gateway-release/home/templates/sandbox-1-3.xml b/gateway-release/home/templates/sandbox-1-3.xml
index ba824e4..9f1ab8f 100644
--- a/gateway-release/home/templates/sandbox-1-3.xml
+++ b/gateway-release/home/templates/sandbox-1-3.xml
@@ -113,9 +113,4 @@
<url>http://localhost:60080</url>
</service>
- <service>
- <role>HIVE</role>
- <url>http://localhost:10000</url>
- </service>
-
</topology>
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml b/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
index b7abde6..d3c52bb 100644
--- a/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
+++ b/gateway-service-oozie/src/main/resources/org/apache/hadoop/gateway/oozie/OozieDeploymentContributor/rewrite.xml
@@ -40,9 +40,16 @@
<rewrite template="{$serviceMappedAddr[JOBTRACKER]}"/>
</rule>
- <rule name="OOZIE/oozie/hdfs-path">
- <match pattern="{path=**}"/>
- <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/{path=**}"/>
+ <rule name="OOZIE/oozie/hdfs-path" flow="OR">
+ <match pattern="/~">
+ <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/user/{$username}"/>
+ </match>
+ <match pattern="/~/{path=**}">
+ <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/user/{$username}/{path=**}"/>
+ </match>
+ <match pattern="{path=**}">
+ <rewrite template="hdfs://{$serviceMappedHost[NAMENODE]}:{$servicePort[NAMENODE]}/{path=**}"/>
+ </match>
</rule>
<filter name="OOZIE/oozie/configuration">
@@ -84,9 +91,15 @@
<detect path="name" value="mapred.input.dir">
<apply path="value" rule="OOZIE/oozie/hdfs-path"/>
</detect>
+ <detect path="name" value="inputDir">
+ <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
+ </detect>
<detect path="name" value="mapred.output.dir">
<apply path="value" rule="OOZIE/oozie/hdfs-path"/>
</detect>
+ <detect path="name" value="outputDir">
+ <apply path="value" rule="OOZIE/oozie/hdfs-path"/>
+ </detect>
</buffer>
</content>
<content type="*/json">
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/34492a25/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
----------------------------------------------------------------------
diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
index d40613e..7116c0d 100644
--- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
+++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/security/SubjectUtils.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.gateway.security;
+import javax.security.auth.Subject;
import java.security.AccessController;
import java.security.Principal;
import java.util.Set;
-import javax.security.auth.Subject;
-
/**
* General utility methods for interrogating the standard java Subject
*/
@@ -65,7 +64,7 @@ public class SubjectUtils {
Set<ImpersonatedPrincipal> impPrincipals = subject.getPrincipals(ImpersonatedPrincipal.class);
if (impPrincipals.size() > 0) {
- return ((PrimaryPrincipal)impPrincipals.toArray()[0]).getName();
+ return ((Principal)impPrincipals.toArray()[0]).getName();
}
return name;