You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by km...@apache.org on 2013/09/03 17:44:13 UTC
git commit: KNOX-52: Support translation of cluster internal
hostnames to external hostnames
Updated Branches:
refs/heads/master a42e12154 -> 003a60c7e
KNOX-52: Support translation of cluster internal hostnames to external hostnames
Project: http://git-wip-us.apache.org/repos/asf/incubator-knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-knox/commit/003a60c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-knox/tree/003a60c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-knox/diff/003a60c7
Branch: refs/heads/master
Commit: 003a60c7eb9b48d83733d8c400f793473d89bde7
Parents: a42e121
Author: Kevin Minder <ke...@hortonworks.com>
Authored: Tue Sep 3 11:42:51 2013 -0400
Committer: Kevin Minder <ke...@hortonworks.com>
Committed: Tue Sep 3 11:42:51 2013 -0400
----------------------------------------------------------------------
.gitignore | 1 +
build.xml | 26 ++++
.../hostmap/impl/HostmapFunctionProcessor.java | 1 +
.../impl/HostmapFunctionProcessorTest.java | 147 +++++++++++++++++++
.../empty-hostmap.txt | 15 ++
.../hdfs-hostmap.txt | 17 +++
.../filter/rewrite/api/UrlRewriteProcessor.java | 2 +
.../impl/UrlRewriteFunctionResolver.java | 8 +-
gateway-release/home/deployments/sample.xml | 13 +-
.../home/samples/ExamplePutFile.groovy | 4 +-
.../home/samples/ExampleSubmitJob.groovy | 4 +-
.../home/samples/ExampleSubmitWorkflow.groovy | 4 +-
.../home/samples/SampleScript.groovy | 4 +-
.../apache/hadoop/gateway/GatewayMessages.java | 2 +-
.../gateway/hdfs/HdfsDeploymentContributor.java | 3 +-
.../hadoop/gateway/GatewayBasicFuncTest.java | 1 +
.../gateway/util/urltemplate/Expander.java | 19 ++-
.../gateway/util/urltemplate/ExpanderTest.java | 45 ++++++
pom.xml | 4 +-
19 files changed, 303 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 2435f6a..e3099b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,7 @@ dependency-reduced-pom.xml
atlassian-ide-plugin.xml
.DS_Store
target
+install
org.apache.hadoop.gateway.security.EmbeddedApacheDirectoryServer/
velocity.log
*.pyc
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 8e7403c..5c32f7f 100644
--- a/build.xml
+++ b/build.xml
@@ -173,4 +173,30 @@
</exec>
</target>
+ <target name="install-test-home" description="Install package release into install directory.">
+ <delete dir="install" quiet="true"/>
+ <unzip src="target/${gateway-version}/${gateway-artifact}-${gateway-version}.zip/" dest="install"/>
+ <echo file="install/${gateway-artifact}-${gateway-version}/conf/security/master">#1.0# Mon, Aug 26 2013 14:25:31.483
+cGkvajhUZHBNSTQ9OjpPMk5PQktYaHRyUmJoTW1zWGo0bytRPT06OkxtVjlvSDdIOWdvSEZqNTRlWVJ2N3c9PQ==</echo>
+ <chmod file="install/${gateway-artifact}-${gateway-version}/conf/security/master" perm="600"/>
+ </target>
+
+ <target name="start-test-ldap">
+ <exec executable="java" dir="install/${gateway-artifact}-${gateway-version}" spawn="true">
+ <arg value="-jar"/>
+ <arg value="bin/ldap.jar"/>
+ <arg value="conf"/>
+ </exec>
+ </target>
+
+ <target name="start-test-gateway">
+ <exec executable="java" dir="install/${gateway-artifact}-${gateway-version}" inputstring="knox\nknox\n">
+ <arg value="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"/>
+ <arg value="-jar"/>
+ <arg value="bin/gateway.jar"/>
+ </exec>
+ </target>
+
+ <target name="start-test-servers" depends="install-test-home,start-test-ldap,start-test-gateway"/>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-hostmap-static/src/main/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessor.java
----------------------------------------------------------------------
diff --git a/gateway-provider-hostmap-static/src/main/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessor.java b/gateway-provider-hostmap-static/src/main/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessor.java
index ba2f64b..9e71a4e 100644
--- a/gateway-provider-hostmap-static/src/main/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessor.java
+++ b/gateway-provider-hostmap-static/src/main/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessor.java
@@ -90,6 +90,7 @@ public class HostmapFunctionProcessor
if( value == null ) {
value = parameter;
}
+// System.out.println( "HOSTMAP: " + parameter + "->" + value );
return value;
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-hostmap-static/src/test/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest.java
----------------------------------------------------------------------
diff --git a/gateway-provider-hostmap-static/src/test/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest.java b/gateway-provider-hostmap-static/src/test/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest.java
index 01178b6..d93d6d1 100644
--- a/gateway-provider-hostmap-static/src/test/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest.java
+++ b/gateway-provider-hostmap-static/src/test/java/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest.java
@@ -66,4 +66,151 @@ public class HostmapFunctionProcessorTest {
assertThat( output.getHost().getFirstValue().getPattern(), is( "test-inbound-rewritten-host" ) );
}
+ @Test
+ public void testHdfsUseCase() throws Exception {
+ URL configUrl = TestUtils.getResourceUrl( this.getClass(), "hdfs-hostmap.txt" );
+
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ EasyMock.expect( environment.getResource( "/WEB-INF/hostmap.txt" ) ).andReturn( configUrl ).anyTimes();
+ Resolver resolver = EasyMock.createNiceMock( Resolver.class );
+ EasyMock.expect( resolver.resolve( "host" ) ).andReturn( Arrays.asList( "test-internal-host" ) ).anyTimes();
+ EasyMock.replay( environment, resolver );
+
+ UrlRewriteRulesDescriptor descriptor = UrlRewriteRulesDescriptorFactory.create();
+ UrlRewriteRuleDescriptor rule = descriptor.addRule( "test-rule" );
+ rule.pattern( "{*}://{host}:{*}/{**}?{**}" );
+ UrlRewriteActionRewriteDescriptorExt rewrite = rule.addStep( "rewrite" );
+ rewrite.template( "{*}://test-static-host:{*}/{**}?server={$hostmap(host)}&{**}" );
+
+ UrlRewriteProcessor rewriter = new UrlRewriteProcessor();
+ rewriter.initialize( environment, descriptor );
+
+ Template input = Parser.parse(
+ "test-scheme://test-external-host:42/test-path/test-file?test-name-1=test-value-1&test-name-2=test-value-2" );
+ Template output = rewriter.rewrite( resolver, input, UrlRewriter.Direction.OUT, "test-rule" );
+ //System.out.println( output );
+ assertThat( output, notNullValue() );
+ assertThat( output.getHost().getFirstValue().getPattern(), is( "test-static-host" ) );
+ assertThat( output.getQuery().get( "server" ).getFirstValue().getPattern(), is( "test-external-host" ) );
+ assertThat( output.getQuery().get( "server" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().get( "test-name-1" ).getFirstValue().getPattern(), is( "test-value-1" ) );
+ assertThat( output.getQuery().get( "test-name-1" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().get( "test-name-2" ).getFirstValue().getPattern(), is( "test-value-2" ) );
+ assertThat( output.getQuery().get( "test-name-2" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().size(), is( 3 ) );
+ }
+
+ @Test
+ public void testQueryToPathRewriteWithFunction() throws Exception {
+ URL configUrl = TestUtils.getResourceUrl( this.getClass(), "hdfs-hostmap.txt" );
+
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ EasyMock.expect( environment.getResource( "/WEB-INF/hostmap.txt" ) ).andReturn( configUrl ).anyTimes();
+ Resolver resolver = EasyMock.createNiceMock( Resolver.class );
+ EasyMock.expect( resolver.resolve( "host" ) ).andReturn( Arrays.asList( "test-internal-host" ) ).anyTimes();
+ EasyMock.replay( environment, resolver );
+
+ UrlRewriteRulesDescriptor descriptor = UrlRewriteRulesDescriptorFactory.create();
+ UrlRewriteRuleDescriptor rule = descriptor.addRule( "test-rule" );
+ rule.pattern( "{*}://{host}:{*}/{**}?{qp1}&{qp2}&{**}" );
+ UrlRewriteActionRewriteDescriptorExt rewrite = rule.addStep( "rewrite" );
+ rewrite.template( "{*}://test-static-host:{*}/{qp1}/{qp2}/{**}?server={$hostmap(host)}&{**}" );
+
+ UrlRewriteProcessor rewriter = new UrlRewriteProcessor();
+ rewriter.initialize( environment, descriptor );
+
+ Template input = Parser.parse(
+ "test-scheme://test-external-host:42/test-path/test-file?qp1=qp1-val&qp2=qp2-val&test-name-1=test-value-1&test-name-2=test-value-2" );
+ Template output = rewriter.rewrite( resolver, input, UrlRewriter.Direction.OUT, "test-rule" );
+ //System.out.println( output );
+ assertThat( output, notNullValue() );
+ assertThat( output.getHost().getFirstValue().getPattern(), is( "test-static-host" ) );
+ assertThat( output.getQuery().get( "server" ).getFirstValue().getPattern(), is( "test-external-host" ) );
+ assertThat( output.getQuery().get( "server" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().get( "test-name-1" ).getFirstValue().getPattern(), is( "test-value-1" ) );
+ assertThat( output.getQuery().get( "test-name-1" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().get( "test-name-2" ).getFirstValue().getPattern(), is( "test-value-2" ) );
+ assertThat( output.getQuery().get( "test-name-2" ).getValues().size(), is( 1 ) );
+ assertThat( output.getQuery().size(), is( 3 ) );
+ }
+
+ @Test
+ public void testUnmappedUseCase() throws Exception {
+ URL configUrl = TestUtils.getResourceUrl( this.getClass(), "hostmap.txt" );
+
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ EasyMock.expect( environment.getResource( "/WEB-INF/hostmap.txt" ) ).andReturn( configUrl ).anyTimes();
+ Resolver resolver = EasyMock.createNiceMock( Resolver.class );
+ EasyMock.expect( resolver.resolve( "host" ) ).andReturn( Arrays.asList( "test-inbound-unmapped-host" ) ).anyTimes();
+ EasyMock.replay( environment, resolver );
+
+ UrlRewriteRulesDescriptor descriptor = UrlRewriteRulesDescriptorFactory.create();
+ UrlRewriteRuleDescriptor rule = descriptor.addRule( "test-rule" );
+ rule.pattern( "{*}://{host}:{*}/{**}?{**}" );
+ UrlRewriteActionRewriteDescriptorExt rewrite = rule.addStep( "rewrite" );
+ rewrite.template( "{*}://{$hostmap(host)}:{*}/{**}?{**}" );
+
+ UrlRewriteProcessor rewriter = new UrlRewriteProcessor();
+ rewriter.initialize( environment, descriptor );
+
+ Template input = Parser.parse(
+ "test-scheme://test-inbound-unmapped-host:42/test-path/test-file?test-name-1=test-value-1&test-name-2=test-value-2" );
+ Template output = rewriter.rewrite( resolver, input, UrlRewriter.Direction.IN, null );
+ //System.out.println( output );
+ assertThat( output, notNullValue() );
+ assertThat( output.getHost().getFirstValue().getPattern(), is( "test-inbound-unmapped-host" ) );
+ }
+
+ @Test
+ public void testMissingFunctionUseCase() throws Exception {
+ URL configUrl = TestUtils.getResourceUrl( this.getClass(), "hostmap.txt" );
+
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ EasyMock.expect( environment.getResource( "/WEB-INF/hostmap.txt" ) ).andReturn( configUrl ).anyTimes();
+ Resolver resolver = EasyMock.createNiceMock( Resolver.class );
+ EasyMock.expect( resolver.resolve( "host" ) ).andReturn( Arrays.asList( "test-inbound-host" ) ).anyTimes();
+ EasyMock.replay( environment, resolver );
+
+ UrlRewriteRulesDescriptor descriptor = UrlRewriteRulesDescriptorFactory.create();
+ UrlRewriteRuleDescriptor rule = descriptor.addRule( "test-rule" );
+ rule.pattern( "{*}://{host}:{*}/{**}?{**}" );
+ UrlRewriteActionRewriteDescriptorExt rewrite = rule.addStep( "rewrite" );
+ rewrite.template( "{*}://{$invalid-function(host)}:{*}/{**}?{**}" );
+
+ UrlRewriteProcessor rewriter = new UrlRewriteProcessor();
+ rewriter.initialize( environment, descriptor );
+
+ Template input = Parser.parse( "test-scheme://test-inbound-host:42/test-path/test-file?test-name=test-value" );
+ Template output = rewriter.rewrite( resolver, input, UrlRewriter.Direction.IN, null );
+ //System.out.println( output );
+ assertThat( output, notNullValue() );
+ assertThat( output.getHost().getFirstValue().getPattern(), is( "test-inbound-host" ) );
+ }
+
+ @Test
+ public void testEmptyHostmapUseCase() throws Exception {
+ URL configUrl = TestUtils.getResourceUrl( this.getClass(), "empty-hostmap.txt" );
+
+ UrlRewriteEnvironment environment = EasyMock.createNiceMock( UrlRewriteEnvironment.class );
+ EasyMock.expect( environment.getResource( "/WEB-INF/hostmap.txt" ) ).andReturn( configUrl ).anyTimes();
+ Resolver resolver = EasyMock.createNiceMock( Resolver.class );
+ EasyMock.expect( resolver.resolve( "host" ) ).andReturn( Arrays.asList( "test-inbound-host" ) ).anyTimes();
+ EasyMock.replay( environment, resolver );
+
+ UrlRewriteRulesDescriptor descriptor = UrlRewriteRulesDescriptorFactory.create();
+ UrlRewriteRuleDescriptor rule = descriptor.addRule( "test-rule" );
+ rule.pattern( "{*}://{host}:{*}/{**}?{**}" );
+ UrlRewriteActionRewriteDescriptorExt rewrite = rule.addStep( "rewrite" );
+ rewrite.template( "{*}://{$invalid-function(host)}:{*}/{**}?{**}" );
+
+ UrlRewriteProcessor rewriter = new UrlRewriteProcessor();
+ rewriter.initialize( environment, descriptor );
+
+ Template input = Parser.parse( "test-scheme://test-inbound-host:42/test-path/test-file?test-name=test-value" );
+ Template output = rewriter.rewrite( resolver, input, UrlRewriter.Direction.IN, null );
+ //System.out.println( output );
+ assertThat( output, notNullValue() );
+ assertThat( output.getHost().getFirstValue().getPattern(), is( "test-inbound-host" ) );
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/empty-hostmap.txt
----------------------------------------------------------------------
diff --git a/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/empty-hostmap.txt b/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/empty-hostmap.txt
new file mode 100644
index 0000000..635f0d9
--- /dev/null
+++ b/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/empty-hostmap.txt
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/hdfs-hostmap.txt
----------------------------------------------------------------------
diff --git a/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/hdfs-hostmap.txt b/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/hdfs-hostmap.txt
new file mode 100644
index 0000000..38eb607
--- /dev/null
+++ b/gateway-provider-hostmap-static/src/test/resources/org/apache/hadoop/gateway/hostmap/impl/HostmapFunctionProcessorTest/hdfs-hostmap.txt
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+test-external-host=test-inbound-internal-host
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessor.java
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessor.java b/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessor.java
index b8f95bf..600b04d 100644
--- a/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessor.java
+++ b/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/api/UrlRewriteProcessor.java
@@ -159,6 +159,8 @@ public class UrlRewriteProcessor implements UrlRewriter {
outputUri = null;
}
}
+// System.out.println( "REWRITE: " + inputUri );
+// System.out.println( " : " + outputUri );
return outputUri;
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/impl/UrlRewriteFunctionResolver.java
----------------------------------------------------------------------
diff --git a/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/impl/UrlRewriteFunctionResolver.java b/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/impl/UrlRewriteFunctionResolver.java
index 0fd8b22..be65591 100644
--- a/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/impl/UrlRewriteFunctionResolver.java
+++ b/gateway-provider-rewrite/src/main/java/org/apache/hadoop/gateway/filter/rewrite/impl/UrlRewriteFunctionResolver.java
@@ -43,6 +43,7 @@ public class UrlRewriteFunctionResolver implements UrlRewriteResolver {
@Override
public String resolve( UrlRewriteContext context, String parameter ) throws Exception {
+// System.out.println( "RESOLVE: " + parameter );
String value = null;
String function = null;
if( parameter != null && parameter.startsWith( "$" ) ) {
@@ -67,7 +68,12 @@ public class UrlRewriteFunctionResolver implements UrlRewriteResolver {
}
}
if( tokenType.equals( TokenType.INDIRECT_PARAMETER ) ) {
- parameter = invokeDelegate( parameter );
+ value = getFirstValue( context.getParameters().resolve( parameter ) );
+ if( value != null ) {
+ parameter = value;
+ } else {
+ parameter = invokeDelegate( parameter );
+ }
}
value = invokeFunction( context, function, parameter );
} else {
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-release/home/deployments/sample.xml
----------------------------------------------------------------------
diff --git a/gateway-release/home/deployments/sample.xml b/gateway-release/home/deployments/sample.xml
index a16c387..e67b2c3 100644
--- a/gateway-release/home/deployments/sample.xml
+++ b/gateway-release/home/deployments/sample.xml
@@ -20,8 +20,8 @@
<gateway>
<provider>
<role>authentication</role>
- <enabled>true</enabled>
<name>ShiroProvider</name>
+ <enabled>true</enabled>
<param>
<name>main.ldapRealm</name>
<value>org.apache.shiro.realm.ldap.JndiLdapRealm</value>
@@ -45,8 +45,17 @@
</provider>
<provider>
<role>identity-assertion</role>
- <enabled>true</enabled>
<name>Pseudo</name>
+ <enabled>true</enabled>
+ </provider>
+ <provider>
+ <role>hostmap</role>
+ <name>static</name>
+ <enabled>true</enabled>
+ <param>
+ <name>localhost</name>
+ <value>sandbox</value>
+ </param>
</provider>
</gateway>
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-release/home/samples/ExamplePutFile.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExamplePutFile.groovy b/gateway-release/home/samples/ExamplePutFile.groovy
index eea4ed1..fa3060a 100644
--- a/gateway-release/home/samples/ExamplePutFile.groovy
+++ b/gateway-release/home/samples/ExamplePutFile.groovy
@@ -21,8 +21,8 @@ import org.apache.hadoop.gateway.shell.Hadoop
import org.apache.hadoop.gateway.shell.hdfs.Hdfs
gateway = "https://localhost:8443/gateway/sample"
-username = "mapred"
-password = "mapred-password"
+username = "bob"
+password = "bob-password"
dataFile = "README"
session = Hadoop.login( gateway, username, password )
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-release/home/samples/ExampleSubmitJob.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleSubmitJob.groovy b/gateway-release/home/samples/ExampleSubmitJob.groovy
index 5a66444..a60053e 100644
--- a/gateway-release/home/samples/ExampleSubmitJob.groovy
+++ b/gateway-release/home/samples/ExampleSubmitJob.groovy
@@ -23,8 +23,8 @@ import org.apache.hadoop.gateway.shell.job.Job
import static java.util.concurrent.TimeUnit.SECONDS
gateway = "https://localhost:8443/gateway/sample"
-username = "mapred"
-password = "mapred-password"
+username = "bob"
+password = "bob-password"
dataFile = "LICENSE"
jarFile = "samples/hadoop-examples.jar"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-release/home/samples/ExampleSubmitWorkflow.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleSubmitWorkflow.groovy b/gateway-release/home/samples/ExampleSubmitWorkflow.groovy
index c28da95..74b641e 100644
--- a/gateway-release/home/samples/ExampleSubmitWorkflow.groovy
+++ b/gateway-release/home/samples/ExampleSubmitWorkflow.groovy
@@ -25,8 +25,8 @@ import static java.util.concurrent.TimeUnit.SECONDS
gateway = "https://localhost:8443/gateway/sample"
jobTracker = "localhost:50300";
nameNode = "localhost:8020";
-username = "mapred"
-password = "mapred-password"
+username = "bob"
+password = "bob-password"
inputFile = "LICENSE"
jarFile = "samples/hadoop-examples.jar"
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-release/home/samples/SampleScript.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/SampleScript.groovy b/gateway-release/home/samples/SampleScript.groovy
index d017646..890f87a 100644
--- a/gateway-release/home/samples/SampleScript.groovy
+++ b/gateway-release/home/samples/SampleScript.groovy
@@ -18,8 +18,8 @@
import org.apache.hadoop.gateway.shell.Hadoop
gateway = "https://localhost:8443/gateway/sample"
-username = "mapred"
-password = "mapred-password"
+username = "bob"
+password = "bob-password"
session = Hadoop.login( gateway, username, password )
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
----------------------------------------------------------------------
diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
index 284c45f..d576038 100644
--- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
+++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java
@@ -145,7 +145,7 @@ public interface GatewayMessages {
void dispatchRequest( String method, URI uri );
@Message( level = MessageLevel.WARN, text = "Connection exception dispatching request: {0} {1}" )
- void dispatchServiceConnectionException( URI uri, @StackTrace(level=MessageLevel.DEBUG) Exception e );
+ void dispatchServiceConnectionException( URI uri, @StackTrace(level=MessageLevel.WARN) Exception e );
@Message( level = MessageLevel.DEBUG, text = "Signature verified: {0}" )
void signatureVerified( boolean verified );
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-service-hdfs/src/main/java/org/apache/hadoop/gateway/hdfs/HdfsDeploymentContributor.java
----------------------------------------------------------------------
diff --git a/gateway-service-hdfs/src/main/java/org/apache/hadoop/gateway/hdfs/HdfsDeploymentContributor.java b/gateway-service-hdfs/src/main/java/org/apache/hadoop/gateway/hdfs/HdfsDeploymentContributor.java
index 50e1280..fa50704 100644
--- a/gateway-service-hdfs/src/main/java/org/apache/hadoop/gateway/hdfs/HdfsDeploymentContributor.java
+++ b/gateway-service-hdfs/src/main/java/org/apache/hadoop/gateway/hdfs/HdfsDeploymentContributor.java
@@ -87,11 +87,10 @@ public class HdfsDeploymentContributor extends ServiceDeploymentContributorBase
rule = rules.addRule( getQualifiedName() + "/datanode/outbound" )
.directions( "outbound" );
-// .pattern( "*://*:*/**?**" );
match = rule.addStep( "match" );
match.pattern( "*://{host}:{port}/{path=**}?{**}" );
rewrite = rule.addStep( "rewrite" );
- rewrite.template( CLUSTER_URL_FUNCTION + DATANODE_EXTERNAL_PATH + "/{path=**}?{host}&{port}&{**}" );
+ rewrite.template( CLUSTER_URL_FUNCTION + DATANODE_EXTERNAL_PATH + "/{path=**}?host={$hostmap(host)}&{port}&{**}" );
rule.addStep( "encode-query" );
UrlRewriteFilterDescriptor filter = rules.addFilter( getQualifiedName() + "/outbound" );
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
index 138479c..e50d2a9 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
@@ -197,6 +197,7 @@ public class GatewayBasicFuncTest {
.addTag( "role" ).addText( "HBASE" )
.addTag( "url" ).addText( driver.getRealUrl( "HBASE" ) )
.gotoRoot();
+ // System.out.println( "GATEWAY=" + xml.toString() );
return xml;
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-util-urltemplate/src/main/java/org/apache/hadoop/gateway/util/urltemplate/Expander.java
----------------------------------------------------------------------
diff --git a/gateway-util-urltemplate/src/main/java/org/apache/hadoop/gateway/util/urltemplate/Expander.java b/gateway-util-urltemplate/src/main/java/org/apache/hadoop/gateway/util/urltemplate/Expander.java
index 56e57e1..2ae5886 100644
--- a/gateway-util-urltemplate/src/main/java/org/apache/hadoop/gateway/util/urltemplate/Expander.java
+++ b/gateway-util-urltemplate/src/main/java/org/apache/hadoop/gateway/util/urltemplate/Expander.java
@@ -165,7 +165,8 @@ public class Expander {
}
Query segment = iterator.next();
String queryName = segment.getQueryName();
- String paramName = segment.getParamName();
+ String funcName = segment.getParamName();
+ String paramName = extractParamNameFromFunction( funcName );
names.remove( paramName );
for( Segment.Value value: segment.getValues() ) {
switch( value.getType() ) {
@@ -181,7 +182,7 @@ public class Expander {
case( Segment.GLOB ):
case( Segment.STAR ):
case( Segment.REGEX ):
- List<String> values = params.resolve( paramName );
+ List<String> values = params.resolve( funcName );
expandQueryValues( segment, queryName, values, builder );
break;
default:
@@ -271,6 +272,20 @@ public class Expander {
}
}
+ private static String extractParamNameFromFunction( String function ) {
+ String param = function;
+ if( param != null && param.startsWith( "$" ) ) {
+ int stop = param.lastIndexOf( ')' );
+ if( stop > 1 ) {
+ int start = param.indexOf( '(' );
+ if( start > -1 ) {
+ param = param.substring( start+1, stop );
+ }
+ }
+ }
+ return param;
+ }
+
private static class EmptyParams implements Params {
@Override
public Set<String> getNames() {
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/gateway-util-urltemplate/src/test/java/org/apache/hadoop/gateway/util/urltemplate/ExpanderTest.java
----------------------------------------------------------------------
diff --git a/gateway-util-urltemplate/src/test/java/org/apache/hadoop/gateway/util/urltemplate/ExpanderTest.java b/gateway-util-urltemplate/src/test/java/org/apache/hadoop/gateway/util/urltemplate/ExpanderTest.java
index f70dfe1..ae65e84 100644
--- a/gateway-util-urltemplate/src/test/java/org/apache/hadoop/gateway/util/urltemplate/ExpanderTest.java
+++ b/gateway-util-urltemplate/src/test/java/org/apache/hadoop/gateway/util/urltemplate/ExpanderTest.java
@@ -271,4 +271,49 @@ public class ExpanderTest {
}
+ @Test
+ public void testExtraParamHandling() throws Exception {
+ Template template;
+ MockParams params;
+ URI expanded;
+
+ params = new MockParams();
+ params.addValue( "scheme", "schemeA" );
+ params.addValue( "host", "hostA" );
+ params.addValue( "query", "queryA" );
+ params.addValue( "query", "queryB" );
+ params.addValue( "path", "pathA" );
+ params.addValue( "path", "pathB" );
+ params.addValue( "extra", "extraA" );
+
+ template = Parser.parse( "{scheme}://host/{path=*]?{query=*}" );
+ expanded = Expander.expand( template, params );
+ assertThat( expanded.toString(), equalTo( "schemeA://host/pathA?query=queryA" ) );
+
+ template = Parser.parse( "{scheme}://host/{path=**}?{query=**}" );
+ expanded = Expander.expand( template, params );
+ assertThat( expanded.toString(), equalTo( "schemeA://host/pathA/pathB?query=queryA&query=queryB" ) );
+
+ template = Parser.parse( "{scheme}://host/{path=**}?{host}&{query=**}&{**}" );
+ expanded = Expander.expand( template, params );
+ assertThat(
+ expanded.toString(),
+ equalTo( "schemeA://host/pathA/pathB?host=hostA&query=queryA&query=queryB&extra=extraA" ) );
+
+ template = Parser.parse( "{scheme}://host/{path=**}?server={host}&{query=**}&{**}" );
+ expanded = Expander.expand( template, params );
+ assertThat(
+ expanded.toString(),
+ equalTo( "schemeA://host/pathA/pathB?server=hostA&query=queryA&query=queryB&extra=extraA" ) );
+
+ // In this case "server-host" is treated as a param name and not found in the params so it
+ // is copied. I'm not really sure what the correct behavior should be. My initial thinking
+ // is that if something within {} isn't resolve to a param it should be dropped from the output.
+ template = Parser.parse( "{scheme}://host/{path=**}?{server=host}&{query=**}&{**}" );
+ expanded = Expander.expand( template, params );
+ assertThat(
+ expanded.toString(),
+ equalTo( "schemeA://host/pathA/pathB?server=host&query=queryA&query=queryB&host=hostA&extra=extraA" ) );
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/003a60c7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 21c3d6a..c33aae1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<packaging>pom</packaging>
<name>gateway</name>
- <description>The Apache Knox Gateway</description>
+ <description>The Apache Knox Project</description>
<url>http://knox.incubator.apache.org/</url>
<modules>
@@ -157,6 +157,7 @@
<exclude>src/vote.txt</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/target/**</exclude>
+ <exclude>**/install/**</exclude>
<exclude>**/test/resources/**</exclude>
<exclude>**/main/resources/**/*.json</exclude>
<exclude>**/home/CHANGES</exclude>
@@ -164,6 +165,7 @@
<exclude>**/home/samples/*.jar</exclude>
<exclude>**/.externalToolBuilders/*</exclude>
<exclude>*.patch</exclude>
+ <exclude>atlassian-ide-plugin.xml</exclude>
</excludes>
</configuration>
</plugin>