You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by co...@apache.org on 2018/02/07 15:02:37 UTC
ranger git commit: RANGER-1940 - Upgrade to Knox 1.0.0
Repository: ranger
Updated Branches:
refs/heads/master f5c15d40b -> 2de803d12
RANGER-1940 - Upgrade to Knox 1.0.0
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/2de803d1
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/2de803d1
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/2de803d1
Branch: refs/heads/master
Commit: 2de803d1241244a38d30cdb762df3b2608b045da
Parents: f5c15d4
Author: Colm O hEigeartaigh <co...@apache.org>
Authored: Wed Feb 7 15:02:14 2018 +0000
Committer: Colm O hEigeartaigh <co...@apache.org>
Committed: Wed Feb 7 15:02:14 2018 +0000
----------------------------------------------------------------------
.../authorization/knox/RangerPDPKnoxFilter.java | 8 +-
.../ranger/services/knox/KnoxRangerTest.java | 142 +++++--------------
pom.xml | 2 +-
.../RangerPDPKnoxDeploymentContributor.java | 12 +-
...gateway.deploy.ProviderDeploymentContributor | 18 ---
...gateway.deploy.ProviderDeploymentContributor | 18 +++
6 files changed, 61 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
----------------------------------------------------------------------
diff --git a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
index 24e8702..f84a3e0 100644
--- a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
+++ b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
@@ -35,10 +35,10 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.gateway.filter.AbstractGatewayFilter;
-import org.apache.hadoop.gateway.security.GroupPrincipal;
-import org.apache.hadoop.gateway.security.ImpersonatedPrincipal;
-import org.apache.hadoop.gateway.security.PrimaryPrincipal;
+import org.apache.knox.gateway.filter.AbstractGatewayFilter;
+import org.apache.knox.gateway.security.GroupPrincipal;
+import org.apache.knox.gateway.security.ImpersonatedPrincipal;
+import org.apache.knox.gateway.security.PrimaryPrincipal;
import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
----------------------------------------------------------------------
diff --git a/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java b/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
index 53e66df..7641921 100644
--- a/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
+++ b/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
@@ -21,26 +21,15 @@ import static io.restassured.RestAssured.given;
import static org.hamcrest.CoreMatchers.is;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.directory.server.protocol.shared.transport.TcpTransport;
-import org.apache.hadoop.gateway.GatewayServer;
-import org.apache.hadoop.gateway.GatewayTestConfig;
-import org.apache.hadoop.gateway.security.ldap.SimpleLdapDirectoryServer;
-import org.apache.hadoop.gateway.services.DefaultGatewayServices;
-import org.apache.hadoop.gateway.services.ServiceLifecycleException;
-import org.apache.hadoop.test.mock.MockServer;
+import org.apache.knox.gateway.GatewayTestConfig;
+import org.apache.knox.gateway.GatewayTestDriver;
import org.apache.http.HttpStatus;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -55,91 +44,25 @@ import io.restassured.response.ValidatableResponse;
*/
public class KnoxRangerTest {
- private static GatewayTestConfig config;
- private static GatewayServer gateway;
- private static SimpleLdapDirectoryServer ldap;
- private static TcpTransport ldapTransport;
- private static MockServer hdfsServer;
- private static MockServer stormServer;
- private static MockServer hbaseServer;
- private static MockServer kafkaServer;
- private static MockServer solrServer;
+ private static GatewayTestDriver driver = new GatewayTestDriver();
@BeforeClass
public static void setupSuite() throws Exception {
- setupLdap();
- hdfsServer = new MockServer( "hdfs", true );
- stormServer = new MockServer( "storm", true );
- hbaseServer = new MockServer( "hbase", true );
- kafkaServer = new MockServer( "kafka", true );
- solrServer = new MockServer( "solr", true );
-
- setupGateway();
+ driver.setResourceBase(KnoxRangerTest.class);
+ driver.setupLdap(0);
+ GatewayTestConfig config = new GatewayTestConfig();
+ driver.setupService("WEBHDFS", "http://localhost:50070/webhdfs", "/cluster/webhdfs", true);
+ driver.setupService("STORM", "http://localhost:8477", "/cluster/storm", true);
+ driver.setupService("SOLR", "http://localhost:8983", "/cluster/solr", true);
+ driver.setupService("WEBHBASE", "http://localhost:60080", "/cluster/hbase", true);
+ driver.setupService("KAFKA", "http://localhost:8477", "/cluster/kafka", true);
+
+ driver.setupGateway(config, "cluster", createTopology(), true);
}
@AfterClass
public static void cleanupSuite() throws Exception {
- gateway.stop();
-
- FileUtils.deleteQuietly( new File( config.getGatewayTopologyDir() ) );
- FileUtils.deleteQuietly( new File( config.getGatewayConfDir() ) );
- FileUtils.deleteQuietly( new File( config.getGatewaySecurityDir() ) );
- FileUtils.deleteQuietly( new File( config.getGatewayDeploymentDir() ) );
- FileUtils.deleteQuietly( new File( config.getGatewayDataDir() ) );
-
- hdfsServer.stop();
- stormServer.stop();
- hbaseServer.stop();
- kafkaServer.stop();
- solrServer.stop();
-
- ldap.stop( true );
- }
-
- private static void setupLdap() throws Exception {
- String basedir = System.getProperty("basedir");
- if (basedir == null) {
- basedir = new File(".").getCanonicalPath();
- }
- Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/users.ldif");
- ldapTransport = new TcpTransport( 0 );
- ldap = new SimpleLdapDirectoryServer( "dc=hadoop,dc=apache,dc=org", path.toFile(), ldapTransport );
- ldap.start();
- }
-
- private static void setupGateway() throws Exception {
-
- File targetDir = new File( System.getProperty( "user.dir" ), "target" );
- File gatewayDir = new File( targetDir, "gateway-home-" + UUID.randomUUID() );
- Assert.assertTrue(gatewayDir.mkdirs());
-
- config = new GatewayTestConfig();
- config.setGatewayHomeDir( gatewayDir.getAbsolutePath() );
-
- config.setGatewayServicesDir(targetDir.getPath() + File.separator + "services");
-
- File topoDir = new File( config.getGatewayTopologyDir() );
- Assert.assertTrue(topoDir.mkdirs());
-
- File deployDir = new File( config.getGatewayDeploymentDir() );
- Assert.assertTrue(deployDir.mkdirs());
-
- File descriptor = new File( topoDir, "cluster.xml" );
- FileOutputStream stream = new FileOutputStream( descriptor );
- createTopology().toStream( stream );
- stream.close();
-
- DefaultGatewayServices srvcs = new DefaultGatewayServices();
- Map<String,String> options = new HashMap<>();
- options.put( "persist-master", "false" );
- options.put( "master", "password" );
- try {
- srvcs.init( config, options );
- } catch ( ServiceLifecycleException e ) {
- e.printStackTrace(); // I18N not required.
- }
-
- gateway = GatewayServer.startGateway( config, srvcs );
+ driver.cleanup();
}
/**
@@ -170,8 +93,7 @@ public class KnoxRangerTest {
.addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent()
.addTag( "param" )
.addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" )
- .addTag( "value" ).addText( "ldap://localhost:" + ldapTransport.getAcceptor().getLocalAddress().getPort() ).gotoParent()
- //.addTag( "value" ).addText(driver.getLdapUrl() ).gotoParent()
+ .addTag( "value" ).addText(driver.getLdapUrl() ).gotoParent()
.addTag( "param" )
.addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism" )
.addTag( "value" ).addText( "simple" ).gotoParent()
@@ -189,19 +111,19 @@ public class KnoxRangerTest {
.gotoRoot()
.addTag("service")
.addTag("role").addText("WEBHDFS")
- .addTag("url").addText("http://localhost:" + hdfsServer.getPort()).gotoParent()
+ .addTag("url").addText(driver.getRealUrl("WEBHDFS")).gotoParent()
.addTag("service")
.addTag("role").addText("STORM")
- .addTag("url").addText("http://localhost:" + stormServer.getPort()).gotoParent()
+ .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent()
.addTag("service")
.addTag("role").addText("WEBHBASE")
- .addTag("url").addText("http://localhost:" + hbaseServer.getPort()).gotoParent()
+ .addTag("url").addText(driver.getRealUrl("WEBHBASE")).gotoParent()
.addTag("service")
.addTag("role").addText("KAFKA")
- .addTag("url").addText("http://localhost:" + kafkaServer.getPort()).gotoParent()
+ .addTag("url").addText(driver.getRealUrl("KAFKA")).gotoParent()
.addTag("service")
.addTag("role").addText("SOLR")
- .addTag("url").addText("http://localhost:" + solrServer.getPort() + "/solr").gotoParent()
+ .addTag("url").addText(driver.getRealUrl("SOLR")).gotoParent()
.gotoRoot();
return xml;
}
@@ -264,7 +186,7 @@ public class KnoxRangerTest {
}
Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/webhdfs-liststatus-test.json");
- hdfsServer
+ driver.getMock("WEBHDFS")
.expect()
.method( "GET" )
.pathInfo( "/v1/hdfstest" )
@@ -280,7 +202,7 @@ public class KnoxRangerTest {
.header("X-XSRF-Header", "jksdhfkhdsf")
.queryParam( "op", "LISTSTATUS" )
.when()
- .get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/webhdfs" + "/v1/hdfstest" )
+ .get( driver.getUrl("WEBHDFS") + "/v1/hdfstest" )
.then()
.statusCode(statusCode)
.log().body();
@@ -297,7 +219,7 @@ public class KnoxRangerTest {
}
Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/cluster-configuration.json");
- stormServer
+ driver.getMock("STORM")
.expect()
.method("GET")
.pathInfo("/api/v1/cluster/configuration")
@@ -310,7 +232,7 @@ public class KnoxRangerTest {
.auth().preemptive().basic(user, password)
.header("X-XSRF-Header", "jksdhfkhdsf")
.header("Accept", "application/json")
- .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/storm" + "/api/v1/cluster/configuration")
+ .when().get( driver.getUrl("STORM") + "/api/v1/cluster/configuration")
.then()
.log().all()
.statusCode(statusCode);
@@ -325,7 +247,7 @@ public class KnoxRangerTest {
Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/webhbase-table-list.xml");
- hbaseServer
+ driver.getMock("WEBHBASE")
.expect()
.method( "GET" )
.pathInfo( "/" )
@@ -340,7 +262,7 @@ public class KnoxRangerTest {
.auth().preemptive().basic( user, password )
.header("X-XSRF-Header", "jksdhfkhdsf")
.header( "Accept", ContentType.XML.toString() )
- .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/hbase" )
+ .when().get( driver.getUrl("WEBHBASE") )
.then()
.statusCode( statusCode )
.log().body();
@@ -348,7 +270,7 @@ public class KnoxRangerTest {
private void makeKafkaInvocation(int statusCode, String user, String password) throws IOException {
- kafkaServer
+ driver.getMock("KAFKA")
.expect()
.method( "GET" )
.pathInfo( "/topics" )
@@ -360,7 +282,7 @@ public class KnoxRangerTest {
.auth().preemptive().basic( user, password )
.header("X-XSRF-Header", "jksdhfkhdsf")
.when()
- .get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/kafka" + "/topics" )
+ .get( driver.getUrl("KAFKA") + "/topics" )
.then()
.statusCode(statusCode)
.log().body();
@@ -374,10 +296,10 @@ public class KnoxRangerTest {
}
Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/query_response.xml");
- solrServer
+ driver.getMock("SOLR")
.expect()
.method("GET")
- .pathInfo("/solr/gettingstarted/select")
+ .pathInfo("/gettingstarted/select")
.queryParam("q", "author_s:William+Shakespeare")
.respond()
.status(HttpStatus.SC_OK)
@@ -388,11 +310,11 @@ public class KnoxRangerTest {
.auth().preemptive().basic(user, password)
.header("X-XSRF-Header", "jksdhfkhdsf")
.header("Accept", "application/json")
- .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/solr"
+ .when().get( driver.getUrl("SOLR")
+ "/gettingstarted/select?q=author_s:William+Shakespeare")
.then()
.log().all()
.statusCode(statusCode);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 550580e..69e042f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -183,7 +183,7 @@
<junit.version>4.12</junit.version>
<kafka.version>0.10.0.0</kafka.version>
<kerby.version>1.0.0</kerby.version>
- <knox.gateway.version>0.13.0</knox.gateway.version>
+ <knox.gateway.version>1.0.0</knox.gateway.version>
<libpam4j.version>1.8</libpam4j.version>
<local.lib.dir>${project.basedir}/../lib/local</local.lib.dir>
<log4j.version>1.2.17</log4j.version>
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java b/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
index 89f8ed2..a5a0865 100644
--- a/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
+++ b/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
@@ -22,12 +22,12 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.hadoop.gateway.deploy.DeploymentContext;
-import org.apache.hadoop.gateway.deploy.ProviderDeploymentContributorBase;
-import org.apache.hadoop.gateway.descriptor.FilterParamDescriptor;
-import org.apache.hadoop.gateway.descriptor.ResourceDescriptor;
-import org.apache.hadoop.gateway.topology.Provider;
-import org.apache.hadoop.gateway.topology.Service;
+import org.apache.knox.gateway.deploy.DeploymentContext;
+import org.apache.knox.gateway.deploy.ProviderDeploymentContributorBase;
+import org.apache.knox.gateway.descriptor.FilterParamDescriptor;
+import org.apache.knox.gateway.descriptor.ResourceDescriptor;
+import org.apache.knox.gateway.topology.Provider;
+import org.apache.knox.gateway.topology.Service;
public class RangerPDPKnoxDeploymentContributor extends ProviderDeploymentContributorBase {
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
deleted file mode 100644
index c0c4576..0000000
--- a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
+++ /dev/null
@@ -1,18 +0,0 @@
-##########################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##########################################################################
-org.apache.ranger.authorization.knox.deploy.RangerPDPKnoxDeploymentContributor
http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
new file mode 100644
index 0000000..c0c4576
--- /dev/null
+++ b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
@@ -0,0 +1,18 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+org.apache.ranger.authorization.knox.deploy.RangerPDPKnoxDeploymentContributor