You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by ka...@apache.org on 2021/07/12 03:33:47 UTC
[submarine] branch master updated: SUBMARINE-833. Support ranger 2.1
This is an automated email from the ASF dual-hosted git repository.
kaihsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 8c33f03 SUBMARINE-833. Support ranger 2.1
8c33f03 is described below
commit 8c33f031868c6de8d8fb5ad4b79faec0807b7c53
Author: sriharsha.tenneti <sr...@grabtaxi.com>
AuthorDate: Sun May 30 21:35:28 2021 +0800
SUBMARINE-833. Support ranger 2.1
### What is this PR for?
Support Ranger 2.1 version
* Split ranger source dir ranger-2 to ranger-2.0 and ranger-2.1 to capture breaking api changes
* Move RangerSparkPlugin to individual ranger-{ranger.major.version} main directories and modify according to ranger version
* Move RangerAdminClientImpl to individual ranger-{ranger.major.version} test directories and modify according to ranger version
* Introduce new profile ranger-2.1 and include elastic search dependency
### What type of PR is it?
[Improvement | Feature ]
### Todos
* Split ranger source dir ranger-2 to ranger-2.0 and ranger-2.1 to capture breaking api changes
* Move RangerSparkPlugin to individual ranger-{ranger.major.version} main directories and modify according to ranger version
* Move RangerAdminClientImpl to individual ranger-{ranger.major.version} test directories and modify according to ranger version
* Introduce new profile ranger-2.1 and include elastic search dependency
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-833
### How should this be tested?
https://travis-ci.org/github/harsha-tenneti/submarine/builds/772933964
<!--
* First time? Setup Travis CI as described on https://submarine.apache.org/contribution/contributions.html#continuous-integration
* Strongly recommended: add automated unit tests for any new or changed behavior
* Outline any manual steps to test the PR here.
-->
### Screenshots (if appropriate)
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? No
* Does this need new documentation? No
Author: sriharsha.tenneti <sr...@grabtaxi.com>
Signed-off-by: Kai-Hsun Chen <ka...@apache.org>
Closes #593 from harsha-tenneti/SUBMARINE-833 and squashes the following commits:
cd55639e [sriharsha.tenneti] SUBMARINE-833. Support ranger 2.1 * Split ranger source dir ranger-2 to ranger-2.0 and ranger-2.1 to capture breaking api changes * Move RangerSparkPlugin to individual ranger-{ranger.major.version} main directories and modify according to ranger version * Move RangerAdminClientImpl to individual ranger-{ranger.major.version} test directories and modify according to ranger version * Introduce new profile ranger-2.1 and include elastic search dependency * A [...]
---
.github/workflows/master.yml | 6 ++--
submarine-security/spark-security/pom.xml | 34 ++++++++++++++++++-
.../RangerSparkPlugin.scala | 0
.../RangerAdminClientImpl.scala | 8 ++---
.../CommandUtils.scala | 0
.../CreateRoleCommand.scala | 0
.../DropRoleCommand.scala | 0
.../ShowCurrentRolesCommand.scala | 0
.../ShowRolesCommand.scala | 0
.../RangerSparkPlugin.scala | 0
.../RangerAdminClientImpl.scala | 8 ++---
.../CommandUtils.scala | 0
.../CreateRoleCommand.scala | 0
.../DropRoleCommand.scala | 0
.../ShowCurrentRolesCommand.scala | 0
.../ShowRolesCommand.scala | 0
.../RangerSparkPlugin.scala | 8 ++---
.../RangerAdminClientImpl.scala | 39 +++++++++++++++-------
18 files changed, 70 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 8e7435c..a89eca7 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -158,11 +158,11 @@ jobs:
kubectl cluster-info
kubectl version
kubectl get pods -n kube-system
-
+
# test-k8s require this config
export KUBECONFIG=~/.kube/kind-config-kind
kind export kubeconfig --kubeconfig ${KUBECONFIG}
-
+
echo "current-context:" $(kubectl config current-context)
echo "environment-kubeconfig:" ${KUBECONFIG}
- name: Download compiled Submarine binaries
@@ -485,7 +485,7 @@ jobs:
strategy:
matrix:
spark-version: ["2.3", "2.4", "3.0"]
- range-version: ["1.2", "2.0"]
+ range-version: ["1.2", "2.0", "2.1"]
exclude:
- spark-version: "2.3"
range-version: "1.2"
diff --git a/submarine-security/spark-security/pom.xml b/submarine-security/spark-security/pom.xml
index 6f79488..0a335d0 100644
--- a/submarine-security/spark-security/pom.xml
+++ b/submarine-security/spark-security/pom.xml
@@ -36,6 +36,7 @@
<properties>
<antlr4.version>4.7</antlr4.version>
<eclipse.jpa.version>2.5.2</eclipse.jpa.version>
+ <elasticsearch.version>7.10.2</elasticsearch.version>
<gson.version>2.2.4</gson.version>
<httpcomponents.httpclient.version>4.5.3</httpcomponents.httpclient.version>
<httpcomponents.httpcore.version>4.4.6</httpcomponents.httpcore.version>
@@ -156,6 +157,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
@@ -340,6 +346,18 @@
</sources>
</configuration>
</execution>
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>ranger-${ranger.major.version}/src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -599,8 +617,22 @@
<httpcomponents.httpcore.version>4.4.6</httpcomponents.httpcore.version>
<httpcomponents.httpmime.version>4.5.3</httpcomponents.httpmime.version>
<ranger.version>2.0.0</ranger.version>
- <ranger.major.version>2</ranger.major.version>
+ <ranger.major.version>2.0</ranger.major.version>
+ <solr.version>7.7.1</solr.version>
+ <gethostname4j.scope>compile</gethostname4j.scope>
+ <jna.scope>compile</jna.scope>
+ </properties>
+ </profile>
+ <profile>
+ <id>ranger-2.1</id>
+ <properties>
+ <httpcomponents.httpclient.version>4.5.3</httpcomponents.httpclient.version>
+ <httpcomponents.httpcore.version>4.4.6</httpcomponents.httpcore.version>
+ <httpcomponents.httpmime.version>4.5.3</httpcomponents.httpmime.version>
+ <ranger.version>2.1.0</ranger.version>
+ <ranger.major.version>2.1</ranger.major.version>
<solr.version>7.7.1</solr.version>
+ <elasticsearch.version>7.10.2</elasticsearch.version>
<gethostname4j.scope>compile</gethostname4j.scope>
<jna.scope>compile</jna.scope>
</properties>
diff --git a/submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala b/submarine-security/spark-security/ranger-1/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
similarity index 100%
copy from submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala
copy to submarine-security/spark-security/ranger-1/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
diff --git a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala b/submarine-security/spark-security/ranger-1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
similarity index 89%
copy from submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
copy to submarine-security/spark-security/ranger-1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
index 9e2b9bd..0723e84 100644
--- a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
+++ b/submarine-security/spark-security/ranger-1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
@@ -44,9 +44,7 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
}
}
- override def getServicePoliciesIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServicePolicies = {
+ override def getServicePoliciesIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServicePolicies = {
policies
}
@@ -54,9 +52,7 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
override def revokeAccess(request: GrantRevokeRequest): Unit = {}
- override def getServiceTagsIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServiceTags = null
+ override def getServiceTagsIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServiceTags = null
override def getTagTypes(tagTypePattern: String): util.List[String] = null
}
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
similarity index 100%
copy from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
similarity index 100%
copy from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
similarity index 100%
copy from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
similarity index 100%
copy from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
similarity index 100%
copy from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
diff --git a/submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala b/submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
similarity index 100%
copy from submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala
copy to submarine-security/spark-security/ranger-2.0/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
diff --git a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala b/submarine-security/spark-security/ranger-2.0/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
similarity index 89%
copy from submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
copy to submarine-security/spark-security/ranger-2.0/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
index 9e2b9bd..0723e84 100644
--- a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
+++ b/submarine-security/spark-security/ranger-2.0/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
@@ -44,9 +44,7 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
}
}
- override def getServicePoliciesIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServicePolicies = {
+ override def getServicePoliciesIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServicePolicies = {
policies
}
@@ -54,9 +52,7 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
override def revokeAccess(request: GrantRevokeRequest): Unit = {}
- override def getServiceTagsIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServiceTags = null
+ override def getServiceTagsIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServiceTags = null
override def getTagTypes(tagTypePattern: String): util.List[String] = null
}
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
similarity index 100%
rename from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/CommandUtils.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
similarity index 100%
rename from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/CreateRoleCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
similarity index 100%
rename from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/DropRoleCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
similarity index 100%
rename from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/ShowCurrentRolesCommand.scala
diff --git a/submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
similarity index 100%
rename from submarine-security/spark-security/ranger-2/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security.command/ShowRolesCommand.scala
diff --git a/submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
similarity index 89%
rename from submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala
rename to submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
index 3e34ab7..e16007c 100644
--- a/submarine-security/spark-security/src/main/scala/org/apache/submarine/spark/security/RangerSparkPlugin.scala
+++ b/submarine-security/spark-security/ranger-2.1/src/main/scala/org.apache.submarine.spark.security/RangerSparkPlugin.scala
@@ -20,20 +20,19 @@
package org.apache.submarine.spark.security
import java.io.{File, IOException}
-
import org.apache.commons.logging.LogFactory
-import org.apache.ranger.authorization.hadoop.config.RangerConfiguration
+import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig
import org.apache.ranger.plugin.service.RangerBasePlugin
object RangerSparkPlugin extends RangerBasePlugin("spark", "sparkSql") {
private val LOG = LogFactory.getLog(RangerSparkPlugin.getClass)
- private val rangerConf: RangerConfiguration = RangerConfiguration.getInstance
+ private val rangerConf: RangerPluginConfig = this.getConfig
val showColumnsOption: String = rangerConf.get(
"xasecure.spark.describetable.showcolumns.authorization.option", "NONE")
- lazy val fsScheme: Array[String] = RangerConfiguration.getInstance()
+ lazy val fsScheme: Array[String] = rangerConf
.get("ranger.plugin.spark.urlauth.filesystem.schemes", "hdfs:,file:")
.split(",")
.map(_.trim)
@@ -56,4 +55,3 @@ object RangerSparkPlugin extends RangerBasePlugin("spark", "sparkSql") {
init()
}
-
diff --git a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala b/submarine-security/spark-security/ranger-2.1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
similarity index 59%
rename from submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
rename to submarine-security/spark-security/ranger-2.1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
index 9e2b9bd..ee442f0 100644
--- a/submarine-security/spark-security/src/test/scala/org/apache/submarine/spark/security/RangerAdminClientImpl.scala
+++ b/submarine-security/spark-security/ranger-2.1/src/test/scala/org.apache.submarine.spark.security/RangerAdminClientImpl.scala
@@ -19,22 +19,23 @@
package org.apache.submarine.spark.security
-import java.nio.file.{Files, FileSystems}
+import java.nio.file.{FileSystems, Files}
import java.util
-
import com.google.gson.GsonBuilder
import org.apache.commons.logging.{Log, LogFactory}
-import org.apache.ranger.admin.client.RangerAdminRESTClient
-import org.apache.ranger.plugin.util.{GrantRevokeRequest, ServicePolicies, ServiceTags}
+import org.apache.hadoop.conf.Configuration
+import org.apache.ranger.admin.client.RangerAdminClient
+import org.apache.ranger.plugin.model.RangerRole
+import org.apache.ranger.plugin.util.{GrantRevokeRequest, GrantRevokeRoleRequest, RangerRoles, RangerUserStore, ServicePolicies, ServiceTags}
-class RangerAdminClientImpl extends RangerAdminRESTClient {
+class RangerAdminClientImpl extends RangerAdminClient {
private val LOG: Log = LogFactory.getLog(classOf[RangerAdminClientImpl])
private val cacheFilename = "sparkSql_hive_jenkins.json"
private val gson =
new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create
private var policies: ServicePolicies = _
- override def init(serviceName: String, appId: String, configPropertyPrefix: String): Unit = {
+ override def init(serviceName: String, appId: String, configPropertyPrefix: String, var4: Configuration): Unit = {
if (policies == null) {
val basedir = this.getClass.getProtectionDomain.getCodeSource.getLocation.getPath
val cachePath = FileSystems.getDefault.getPath(basedir, cacheFilename)
@@ -44,9 +45,7 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
}
}
- override def getServicePoliciesIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServicePolicies = {
+ override def getServicePoliciesIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServicePolicies = {
policies
}
@@ -54,9 +53,25 @@ class RangerAdminClientImpl extends RangerAdminRESTClient {
override def revokeAccess(request: GrantRevokeRequest): Unit = {}
- override def getServiceTagsIfUpdated(
- lastKnownVersion: Long,
- lastActivationTimeInMillis: Long): ServiceTags = null
+ override def getServiceTagsIfUpdated(lastKnownVersion: Long, lastActivationTimeInMillis: Long): ServiceTags = null
override def getTagTypes(tagTypePattern: String): util.List[String] = null
+
+ override def getRolesIfUpdated(l: Long, l1: Long): RangerRoles = null
+
+ override def createRole(rangerRole: RangerRole): RangerRole = null
+
+ override def dropRole(s: String, s1: String): Unit = {}
+
+ override def getAllRoles(s: String): util.List[String] = null
+
+ override def getUserRoles(s: String): util.List[String] = null
+
+ override def getRole(s: String, s1: String): RangerRole = null
+
+ override def grantRole(grantRevokeRoleRequest: GrantRevokeRoleRequest): Unit = {}
+
+ override def revokeRole(grantRevokeRoleRequest: GrantRevokeRoleRequest): Unit = {}
+
+ override def getUserStoreIfUpdated(l: Long, l1: Long): RangerUserStore = null
}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org