You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@knox.apache.org by su...@apache.org on 2015/01/27 17:26:33 UTC
[07/14] knox git commit: KNOX-483 added webhdfs sans ha
KNOX-483 added webhdfs sans ha
Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/feae9e98
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/feae9e98
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/feae9e98
Branch: refs/heads/KNOX-481
Commit: feae9e98449bd3174aa9504622b2ec8ced78c436
Parents: 617c829
Author: Sumit Gupta <su...@apache.org>
Authored: Mon Jan 19 12:21:03 2015 -0500
Committer: Sumit Gupta <su...@apache.org>
Committed: Tue Jan 27 11:25:20 2015 -0500
----------------------------------------------------------------------
.../services/webhdfs/2.4.0/rewrite.xml | 70 ++++++++++++++++++++
.../services/webhdfs/2.4.0/service.xml | 38 +++++++++++
.../services/yarn-rm/2.5.0/service.xml | 2 -
.../hdfs/WebHdfsDeploymentContributor.java | 10 +--
.../deploy/DeploymentFactoryFuncTest.java | 26 +++++++-
5 files changed, 137 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/knox/blob/feae9e98/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
new file mode 100644
index 0000000..efbd93d
--- /dev/null
+++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<rules>
+
+ <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="hdfs://*:*/{path=**}?{**}">
+ <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="webhdfs://*:*/{path=**}?{**}">
+ <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="OUT" name="WEBHDFS/webhdfs/outbound/namenode/headers/location">
+ <match pattern="{scheme}://{host}:{port}/{path=**}?{**}"/>
+ <rewrite template="{$frontend[url]}/webhdfs/data/v1/{path=**}?{scheme}?host={$hostmap(host)}?{port}?{**}"/>
+ <encrypt-query/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/hdfs" pattern="hdfs:/{path=**}?{**}">
+ <rewrite template="{$serviceMappedUrl[NAMENODE]}/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/webhdfs" pattern="webhdfs:/{path=**}?{**}">
+ <rewrite template="{$serviceUrl[WEBHDFS]}/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/root" pattern="*://*:*/**/webhdfs/{version}/?{**}">
+ <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/file" pattern="*://*:*/**/webhdfs/{version}/{path=**}?{**}">
+ <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home" pattern="*://*:*/**/webhdfs/{version}/~?{**}">
+ <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home/file" pattern="*://*:*/**/webhdfs/{version}/~/{path=**}?{**}">
+ <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}/{path=**}?{**}"/>
+ </rule>
+
+ <rule dir="IN" name="WEBHDFS/webhdfs/inbound/datanode">
+ <decrypt-query/>
+ <match pattern="*://*:*/**/webhdfs/data/*/{path=**}?{scheme}?{host}?{port}?{**}"/>
+ <rewrite template="{scheme}://{host}:{port}/{path=**}?{**}"/>
+ </rule>
+
+ <filter name="WEBHDFS/webhdfs/outbound/namenode/headers">
+ <content type="application/x-http-headers">
+ <apply path="Location" rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/>
+ </content>
+ </filter>
+
+</rules>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/knox/blob/feae9e98/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
new file mode 100644
index 0000000..1aeaf70
--- /dev/null
+++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<service role="WEBHDFS" name="webhdfs" version="2.4.0">
+ <urls>
+ <url pattern="/webhdfs/v1/?**">
+ <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/root" apply-to="request.url"/>
+ </url>
+ <url pattern="/webhdfs/v1/**?**">
+ <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/file" apply-to="request.url"/>
+ <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/>
+ </url>
+ <url pattern="/webhdfs/v1/~?**">
+ <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home" apply-to="request.url"/>
+ </url>
+ <url pattern="/webhdfs/v1/~/**?**">
+ <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home/file" apply-to="request.url"/>
+ <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/>
+ </url>
+ <url pattern="/webhdfs/data/v1/**?**">
+ <rewrite-filter ref="WEBHDFS/webhdfs/inbound/datanode" apply-to="request.url"/>
+ </url>
+ </urls>
+</service>
http://git-wip-us.apache.org/repos/asf/knox/blob/feae9e98/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
----------------------------------------------------------------------
diff --git a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
index 61a77f0..f77d94d 100644
--- a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
+++ b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
@@ -16,13 +16,11 @@
limitations under the License.
-->
<service role="RESOURCEMANAGER" name="resourcemanager" version="2.5.0">
- <policies></policies>
<urls>
<url pattern="/resourcemanager/v1/cluster/"/>
<url pattern="/resourcemanager/v1/cluster/**?**"/>
<url pattern="/resourcemanager/v1/cluster/apps?**">
<rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>
- <policies></policies>
</url>
<url pattern="/resourcemanager/v1/cluster/apps?**">
<rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>
http://git-wip-us.apache.org/repos/asf/knox/blob/feae9e98/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
----------------------------------------------------------------------
diff --git a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
index 0e63597..295e081 100644
--- a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
+++ b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
@@ -44,19 +44,19 @@ public class WebHdfsDeploymentContributor extends ServiceDeploymentContributorBa
@Override
public String getRole() {
- return "WEBHDFS";
+ return "xWEBHDFS";
}
@Override
public String getName() {
- return "webhdfs";
+ return "xwebhdfs";
}
@Override
public void contributeService( DeploymentContext context, Service service ) throws Exception {
- contributeRewriteRules( context, service );
- contributeNameNodeResource( context, service );
- contributeDataNodeResource( context, service );
+// contributeRewriteRules( context, service );
+// contributeNameNodeResource( context, service );
+// contributeDataNodeResource( context, service );
}
private void contributeRewriteRules( DeploymentContext context, Service service ) throws URISyntaxException, IOException {
http://git-wip-us.apache.org/repos/asf/knox/blob/feae9e98/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
index e2c1bae..8f8f236 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.gateway.deploy;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.gateway.GatewayTestConfig;
import org.apache.hadoop.gateway.config.GatewayConfig;
import org.apache.hadoop.gateway.services.DefaultGatewayServices;
@@ -68,6 +69,7 @@ public class DeploymentFactoryFuncTest {
// ((GatewayTestConfig) config).setDeploymentDir( "clusters" );
+ addStacksDir(config, gatewayDir);
DefaultGatewayServices srvcs = new DefaultGatewayServices();
Map<String,String> options = new HashMap<String,String>();
options.put("persist-master", "false");
@@ -121,6 +123,7 @@ public class DeploymentFactoryFuncTest {
((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() );
File deployDir = new File( config.getGatewayDeploymentDir() );
deployDir.mkdirs();
+ addStacksDir(config, gatewayDir);
DefaultGatewayServices srvcs = new DefaultGatewayServices();
Map<String,String> options = new HashMap<String,String>();
@@ -175,6 +178,7 @@ public class DeploymentFactoryFuncTest {
((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() );
File deployDir = new File( config.getGatewayDeploymentDir() );
deployDir.mkdirs();
+ addStacksDir(config, gatewayDir);
DefaultGatewayServices srvcs = new DefaultGatewayServices();
Map<String,String> options = new HashMap<String,String>();
@@ -214,8 +218,8 @@ public class DeploymentFactoryFuncTest {
topology.addProvider( authorizer );
WebArchive war = DeploymentFactory.createDeployment( config, topology );
- //File dir = new File( System.getProperty( "user.dir" ) );
- //File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war" );
+// File dir = new File( System.getProperty( "user.dir" ) );
+// File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war" );
Document web = parse( war.get( "WEB-INF/web.xml" ).getAsset().openStream() );
assertThat( web, hasXPath( "/web-app/servlet/servlet-name", equalTo( "test-cluster" ) ) );
@@ -363,6 +367,24 @@ public class DeploymentFactoryFuncTest {
return builder.parse( source );
}
+ private void addStacksDir(GatewayConfig config, File targetDir) {
+ File stacksDir = new File( config.getGatewayStacksDir() );
+ stacksDir.mkdirs();
+ //TODO: [sumit] This is a hack for now, need to find a better way to locate the source resources for 'stacks' to be tested
+ String pathToStacksSource = "gateway-service-definitions/src/main/resources/services";
+ File stacksSourceDir = new File( targetDir.getParent(), pathToStacksSource);
+ if (!stacksSourceDir.exists()) {
+ stacksSourceDir = new File( targetDir.getParentFile().getParent(), pathToStacksSource);
+ }
+ if (stacksSourceDir.exists()) {
+ try {
+ FileUtils.copyDirectoryToDirectory(stacksSourceDir, stacksDir);
+ } catch ( IOException e) {
+ fail(e.getMessage());
+ }
+ }
+
+ }
// private void dump( Document document ) throws TransformerException {
// Transformer transformer = TransformerFactory.newInstance().newTransformer();
// transformer.setOutputProperty( OutputKeys.INDENT, "yes" );