You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2020/06/01 11:08:54 UTC
[ranger] branch master updated (8050020 -> 5d1f87c)
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git.
from 8050020 RANGER-2839: Assorted improvements for debugging and handling of thread terminations, clean-up of unused data, etc
new 5dc7509 RANGER-2834: Upgrade Ranger to support Elasticsearch 7.6.0
new 5d1f87c RANGER-2840 : Create policy with non existing group is failing due to multiple threads creating same group
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
agents-audit/pom.xml | 121 +++++++++-
.../destination/ElasticSearchAuditDestination.java | 5 +-
.../authorizer/RangerElasticsearchAuthorizer.java | 6 +-
.../elasticsearch/RangerServiceElasticsearch.java | 5 +-
.../elasticsearch/client/ElasticsearchClient.java | 12 +-
pom.xml | 7 +-
.../authorizer/RangerElasticsearchAuthorizer.java | 10 +-
.../plugin/RangerElasticsearchPlugin.java | 12 +-
.../action/filter/RangerSecurityActionFilter.java | 30 ++-
.../rest/filter/RangerSecurityRestFilter.java | 30 ++-
.../elasticsearch/plugin/utils/RequestUtils.java | 5 +-
.../org/apache/ranger/biz/PolicyRefUpdater.java | 254 ++++++++++++++++++---
.../ElasticSearchAccessAuditsService.java | 2 +-
.../ranger/elasticsearch/ElasticSearchUtil.java | 5 +-
14 files changed, 421 insertions(+), 83 deletions(-)
[ranger] 01/02: RANGER-2834: Upgrade Ranger to support
Elasticsearch 7.6.0
Posted by pr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 5dc750994aa0164d519aab709dd28789164b756b
Author: pradeep <pr...@apache.org>
AuthorDate: Wed May 27 19:27:26 2020 +0530
RANGER-2834: Upgrade Ranger to support Elasticsearch 7.6.0
---
agents-audit/pom.xml | 121 ++++++++++++++++++++-
.../destination/ElasticSearchAuditDestination.java | 5 +-
.../authorizer/RangerElasticsearchAuthorizer.java | 6 +-
.../elasticsearch/RangerServiceElasticsearch.java | 5 +-
.../elasticsearch/client/ElasticsearchClient.java | 12 +-
pom.xml | 7 +-
.../authorizer/RangerElasticsearchAuthorizer.java | 10 +-
.../plugin/RangerElasticsearchPlugin.java | 12 +-
.../action/filter/RangerSecurityActionFilter.java | 30 +++--
.../rest/filter/RangerSecurityRestFilter.java | 30 +++--
.../elasticsearch/plugin/utils/RequestUtils.java | 5 +-
.../ElasticSearchAccessAuditsService.java | 2 +-
.../ranger/elasticsearch/ElasticSearchUtil.java | 5 +-
13 files changed, 198 insertions(+), 52 deletions(-)
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 12d621f..b30474a 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -149,6 +149,36 @@
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-x-content</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-secure-sm</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>parent-join-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>aggs-matrix-stats-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>rank-eval-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.plugin</groupId>
+ <artifactId>lang-mustache-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore-nio</artifactId>
<version>${httpcomponents.httpcore.version}</version>
@@ -163,6 +193,95 @@
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-backward-codecs</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-grouping</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-misc</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-sandbox</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial3d</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial-extras</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-suggest</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda.time.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>${hppc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${log4j.core.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${log4j.core.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
index dc00fec..bda582a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
@@ -42,6 +42,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
@@ -121,7 +122,7 @@ public class ElasticSearchAuditDestination extends AuditDestination {
addFailedCount(eventList.size());
logFailedEvent(eventList, ex);
}
- BulkResponse response = client.bulk(bulkRequest);
+ BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (response.status().getStatus() >= 400) {
addFailedCount(eventList.size());
logFailedEvent(eventList, "HTTP " + response.status().getStatus());
@@ -197,7 +198,7 @@ public class ElasticSearchAuditDestination extends AuditDestination {
LOG.debug("Initialized client");
boolean exits = false;
try {
- exits = restHighLevelClient.indices().open(new OpenIndexRequest(this.index)).isShardsAcknowledged();
+ exits = restHighLevelClient.indices().open(new OpenIndexRequest(this.index), RequestOptions.DEFAULT).isShardsAcknowledged();
} catch (Exception e) {
LOG.warn("Error validating index " + this.index);
}
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
index bd0585c..f5201ce 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.Logger;
import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
@@ -30,13 +29,14 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResult;
import org.apache.ranger.plugin.service.RangerBasePlugin;
import org.apache.ranger.services.elasticsearch.client.ElasticsearchResourceMgr;
import org.apache.ranger.services.elasticsearch.privilege.IndexPrivilegeUtils;
-import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
public class RangerElasticsearchAuthorizer implements RangerElasticsearchAccessControl {
- private static final Logger LOG = ESLoggerFactory.getLogger(RangerElasticsearchAuthorizer.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizer.class);
private static volatile RangerElasticsearchInnerPlugin elasticsearchPlugin = null;
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
index ed1c98e..100851d 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
@@ -22,16 +22,17 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerServiceDef;
import org.apache.ranger.plugin.service.RangerBaseService;
import org.apache.ranger.plugin.service.ResourceLookupContext;
import org.apache.ranger.services.elasticsearch.client.ElasticsearchResourceMgr;
+import org.slf4j.LoggerFactory;
public class RangerServiceElasticsearch extends RangerBaseService {
- private static final Logger LOG = Logger.getLogger(RangerServiceElasticsearch.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerServiceElasticsearch.class);
public RangerServiceElasticsearch() {
super();
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/client/ElasticsearchClient.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/client/ElasticsearchClient.java
index d5c170d..598dabd 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/client/ElasticsearchClient.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/client/ElasticsearchClient.java
@@ -45,8 +45,6 @@ import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
public class ElasticsearchClient extends BaseClient {
@@ -134,7 +132,6 @@ public class ElasticsearchClient extends BaseClient {
ClientResponse response = null;
Client client = Client.create();
- client.addFilter(new HTTPBasicAuthFilter(userName, StringUtils.EMPTY));
for (String currentUrl : elasticsearchUrls) {
if (StringUtils.isBlank(currentUrl)) {
continue;
@@ -142,7 +139,7 @@ public class ElasticsearchClient extends BaseClient {
String url = currentUrl.trim() + elasticsearchApi;
try {
- response = getClientResponse(url, client);
+ response = getClientResponse(url, client, userName);
if (response != null) {
if (response.getStatus() == HttpStatus.SC_OK) {
@@ -161,14 +158,13 @@ public class ElasticsearchClient extends BaseClient {
return response;
}
- private static ClientResponse getClientResponse(String url, Client client) {
+ private static ClientResponse getClientResponse(String url, Client client, String userName) {
if (LOG.isDebugEnabled()) {
LOG.debug("getClientResponse():calling " + url);
}
- WebResource webResource = client.resource(url);
-
- ClientResponse response = webResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ ClientResponse response = client.resource(url).accept(MediaType.APPLICATION_JSON).
+ header("userName", userName).get(ClientResponse.class);
if (response != null) {
if (LOG.isDebugEnabled()) {
diff --git a/pom.xml b/pom.xml
index 5605a17..d100bdd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,7 +113,7 @@
<curator.test.version>2.12.0</curator.test.version>
<derby.version>10.11.1.1</derby.version>
<eclipse.jpa.version>2.5.2</eclipse.jpa.version>
- <elasticsearch.version>6.2.2</elasticsearch.version>
+ <elasticsearch.version>7.6.0</elasticsearch.version>
<enunciate.version>2.11.1</enunciate.version>
<findbugs.plugin.version>3.0.3</findbugs.plugin.version>
<google.guava.version>25.1-jre</google.guava.version>
@@ -217,7 +217,10 @@
<net.minidev.asm.version>1.0.2</net.minidev.asm.version>
<org.bouncycastle.bcprov-jdk15on>1.59</org.bouncycastle.bcprov-jdk15on>
<org.bouncycastle.bcpkix-jdk15on>1.59</org.bouncycastle.bcpkix-jdk15on>
- <lucene.version>7.2.1</lucene.version>
+ <lucene.version>8.4.0</lucene.version>
+ <hppc.version>0.8.0</hppc.version>
+ <joda.time.version>2.10.5</joda.time.version>
+ <log4j.core.version>2.11.1</log4j.core.version>
</properties>
<profiles>
<profile>
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
index 8f33cb9..6f049dd 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
@@ -19,18 +19,18 @@ package org.apache.ranger.authorization.elasticsearch.authorizer;
import java.util.List;
-import org.apache.logging.log4j.Logger;
import org.apache.ranger.plugin.classloader.RangerPluginClassLoader;
-import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class RangerElasticsearchAuthorizer {
- private static final Logger LOG = ESLoggerFactory.getLogger(RangerElasticsearchAuthorizer.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizer.class);
private static final String RANGER_PLUGIN_TYPE = "elasticsearch";
private static final String RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME = "org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer";
-
private static RangerPluginClassLoader rangerPluginClassLoader = null;
private static ClassLoader esClassLoader = null;
@@ -67,9 +67,7 @@ public class RangerElasticsearchAuthorizer {
@SuppressWarnings("unchecked")
Class<RangerElasticsearchAccessControl> cls = (Class<RangerElasticsearchAccessControl>) Class
.forName(RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME, true, rangerPluginClassLoader);
-
activatePluginClassLoader();
-
rangerElasticsearchAccessControl = cls.newInstance();
} catch (Exception e) {
LOG.error("Error Enabling RangerElasticsearchAuthorizer", e);
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
index faeac52..b196b0f 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
@@ -27,14 +27,12 @@ import java.util.Collections;
import java.util.List;
import java.util.function.UnaryOperator;
-import org.apache.logging.log4j.Logger;
import org.apache.ranger.authorization.elasticsearch.plugin.action.filter.RangerSecurityActionFilter;
import org.apache.ranger.authorization.elasticsearch.plugin.rest.filter.RangerSecurityRestFilter;
import org.elasticsearch.action.support.ActionFilter;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -46,10 +44,12 @@ import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
- private static final Logger LOG = ESLoggerFactory.getLogger(RangerElasticsearchPlugin.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchPlugin.class);
private static final String RANGER_ELASTICSEARCH_PLUGIN_CONF_NAME = "ranger-elasticsearch-plugin";
@@ -59,6 +59,7 @@ public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
public RangerElasticsearchPlugin(Settings settings) {
this.settings = settings;
+ LOG.debug("settings:"+this.settings);
}
@Override
@@ -68,8 +69,7 @@ public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
@Override
public UnaryOperator<RestHandler> getRestHandlerWrapper(ThreadContext threadContext) {
- return (UnaryOperator<RestHandler>) (handler -> new RangerSecurityRestFilter(this.settings, threadContext,
- handler));
+ return (UnaryOperator<RestHandler>) (handler -> new RangerSecurityRestFilter(threadContext, handler));
}
@Override
@@ -81,7 +81,7 @@ public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
addPluginConfig2Classpath(environment);
- rangerSecurityActionFilter = new RangerSecurityActionFilter(this.settings, threadPool.getThreadContext());
+ rangerSecurityActionFilter = new RangerSecurityActionFilter(threadPool.getThreadContext());
return Collections.singletonList(rangerSecurityActionFilter);
}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
index 60baaea..945d6a1 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
@@ -20,7 +20,6 @@ package org.apache.ranger.authorization.elasticsearch.plugin.action.filter;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.Logger;
import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer;
import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken;
import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils;
@@ -30,23 +29,23 @@ import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.support.ActionFilter;
import org.elasticsearch.action.support.ActionFilterChain;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.tasks.Task;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class RangerSecurityActionFilter extends AbstractComponent implements ActionFilter {
+public class RangerSecurityActionFilter extends AbstractLifecycleComponent implements ActionFilter {
- private static final Logger LOG = ESLoggerFactory.getLogger(RangerSecurityActionFilter.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerSecurityActionFilter.class);
private final ThreadContext threadContext;
private final RangerElasticsearchAuthorizer rangerElasticsearchAuthorizer = new RangerElasticsearchAuthorizer();
- public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext) {
- super(settings);
+ public RangerSecurityActionFilter(ThreadContext threadContext) {
+ super();
this.threadContext = threadContext;
}
@@ -79,4 +78,19 @@ public class RangerSecurityActionFilter extends AbstractComponent implements Act
}
chain.proceed(task, action, request, listener);
}
+
+ @Override
+ protected void doStart() {
+
+ }
+
+ @Override
+ protected void doStop() {
+
+ }
+
+ @Override
+ protected void doClose() {
+
+ }
}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java
index 117356a..92215e5 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/rest/filter/RangerSecurityRestFilter.java
@@ -18,31 +18,30 @@
package org.apache.ranger.authorization.elasticsearch.plugin.rest.filter;
import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.Logger;
import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken;
import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.logging.ESLoggerFactory;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class RangerSecurityRestFilter extends AbstractComponent implements RestHandler {
+public class RangerSecurityRestFilter extends AbstractLifecycleComponent implements RestHandler {
- private static final Logger LOG = ESLoggerFactory.getLogger(RangerSecurityRestFilter.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RangerSecurityRestFilter.class);
private final RestHandler restHandler;
private final ThreadContext threadContext;
- public RangerSecurityRestFilter(final Settings settings, final ThreadContext threadContext,
+ public RangerSecurityRestFilter(final ThreadContext threadContext,
final RestHandler restHandler) {
- super(settings);
+ super();
this.restHandler = restHandler;
this.threadContext = threadContext;
}
@@ -70,4 +69,19 @@ public class RangerSecurityRestFilter extends AbstractComponent implements RestH
this.restHandler.handleRequest(request, channel, client);
}
+
+ @Override
+ protected void doStart() {
+
+ }
+
+ @Override
+ protected void doStop() {
+
+ }
+
+ @Override
+ protected void doClose() {
+
+ }
}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/utils/RequestUtils.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/utils/RequestUtils.java
index 8a04b06..6786c14 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/utils/RequestUtils.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/utils/RequestUtils.java
@@ -44,7 +44,7 @@ public class RequestUtils {
public static final String CLIENT_IP_ADDRESS = "ClientIPAddress";
public static String getClientIPAddress(RestRequest request) {
- SocketAddress socketAddress = request.getRemoteAddress();
+ SocketAddress socketAddress = request.getHttpChannel().getRemoteAddress();
if (socketAddress instanceof InetSocketAddress) {
return ((InetSocketAddress) socketAddress).getAddress().getHostAddress();
}
@@ -98,8 +98,7 @@ public class RequestUtils {
}
if (request instanceof BulkRequest) {
- @SuppressWarnings("rawtypes")
- List<DocWriteRequest> requests = ((BulkRequest) request).requests();
+ @SuppressWarnings("rawtypes") List<DocWriteRequest<?>> requests = ((BulkRequest) request).requests();
if (CollectionUtils.isNotEmpty(requests)) {
for (DocWriteRequest<?> docWriteRequest : requests) {
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
index c88726f..f28eb24 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java
@@ -121,7 +121,7 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu
VXAccessAuditList returnList = new VXAccessAuditList();
returnList.setPageSize(searchCriteria.getMaxRows());
returnList.setResultSize(response.getHits().getHits().length);
- returnList.setTotalCount(response.getHits().getTotalHits());
+ returnList.setTotalCount(response.getHits().getTotalHits().value);
returnList.setStartIndex(searchCriteria.getStartIndex());
returnList.setVXAccessAudits(xAccessAuditList);
return returnList;
diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
index 32a4583..9bee640 100644
--- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.get.MultiGetItemResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@@ -143,7 +144,7 @@ public class ElasticSearchUtil {
SearchRequest query = new SearchRequest();
query.indices(index);
query.source(searchSourceBuilder.query(boolQueryBuilder));
- return client.search(query);
+ return client.search(query, RequestOptions.DEFAULT);
}
public void setSortClause(SearchCriteria searchCriteria,
@@ -230,7 +231,7 @@ public class ElasticSearchUtil {
item.fetchSourceContext(FetchSourceContext.FETCH_SOURCE);
multiGetRequest.add(item);
}
- return client.multiGet(multiGetRequest).getResponses();
+ return client.multiGet(multiGetRequest, RequestOptions.DEFAULT).getResponses();
}
private class QueryAccumulator {
[ranger] 02/02: RANGER-2840 : Create policy with non existing group
is failing due to multiple threads creating same group
Posted by pr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pradeep pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 5d1f87cd92eeed7cc7deb3fd0f7e688ef71e9250
Author: Dineshkumar Yadav <di...@outlook.com>
AuthorDate: Fri May 29 16:30:44 2020 +0530
RANGER-2840 : Create policy with non existing group is failing due to multiple threads creating same group
Signed-off-by: pradeep <pr...@apache.org>
---
.../org/apache/ranger/biz/PolicyRefUpdater.java | 254 ++++++++++++++++++---
1 file changed, 223 insertions(+), 31 deletions(-)
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
index f978d5d..564a2e7 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/PolicyRefUpdater.java
@@ -28,6 +28,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.common.RangerCommonEnums;
+import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXAccessTypeDef;
import org.apache.ranger.entity.XXDataMaskTypeDef;
@@ -44,6 +45,7 @@ import org.apache.ranger.entity.XXPolicyRefUser;
import org.apache.ranger.entity.XXResourceDef;
import org.apache.ranger.entity.XXRole;
import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.entity.XXTrxLog;
import org.apache.ranger.entity.XXUser;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicy.RangerDataMaskPolicyItem;
@@ -53,6 +55,8 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemDataMaskInfo;
import org.apache.ranger.plugin.model.RangerRole;
import org.apache.ranger.service.RangerAuditFields;
+import org.apache.ranger.service.RangerTransactionService;
+import org.apache.ranger.service.XGroupService;
import org.apache.ranger.service.XUserService;
import org.apache.ranger.view.VXGroup;
import org.springframework.beans.factory.annotation.Autowired;
@@ -83,6 +87,15 @@ public class PolicyRefUpdater {
@Autowired
RangerBizUtil rangerBizUtil;
+ @Autowired
+ XGroupService xGroupService;
+
+ @Autowired
+ RangerTransactionSynchronizationAdapter rangerTransactionSynchronizationAdapter;
+
+ @Autowired
+ RangerTransactionService transactionService;
+
public void createNewPolMappingForRefTable(RangerPolicy policy, XXPolicy xPolicy, XXServiceDef xServiceDef) throws Exception {
if(policy == null) {
return;
@@ -180,7 +193,6 @@ public class PolicyRefUpdater {
RangerBizUtil.setBulkMode(oldBulkMode);
daoMgr.getXXPolicyRefRole().batchCreate(xPolRoles);
- List<XXPolicyRefGroup> xPolGroups = new ArrayList<>();
for (String group : groupNames) {
if (StringUtils.isBlank(group)) {
continue;
@@ -190,24 +202,13 @@ public class PolicyRefUpdater {
Long groupId = null;
if (xGroup != null) {
groupId = xGroup.getId();
+ groupPolicyAssociation(xPolicy,groupId,group );
}
else {
- RangerBizUtil.setBulkMode(false);
- groupId = createGroupForPolicy(group);
+ createGroupForPolicy(group, xPolicy);
}
-
- XXPolicyRefGroup xPolGroup = rangerAuditFields.populateAuditFields(new XXPolicyRefGroup(), xPolicy);
-
- xPolGroup.setPolicyId(policy.getId());
- xPolGroup.setGroupId(groupId);
- xPolGroup.setGroupName(group);
-
- xPolGroups.add(xPolGroup);
}
- RangerBizUtil.setBulkMode(oldBulkMode);
- daoMgr.getXXPolicyRefGroup().batchCreate(xPolGroups);
- List<XXPolicyRefUser> xPolUsers = new ArrayList<>();
for (String user : userNames) {
if (StringUtils.isBlank(user)) {
continue;
@@ -217,23 +218,13 @@ public class PolicyRefUpdater {
Long userId = null;
if(xUser != null){
userId = xUser.getId();
+ userPolicyAssociation(xPolicy,userId, user );
}
else {
- RangerBizUtil.setBulkMode(false);
-
- userId = createUserForPolicy(user);
+ createUserForPolicy(user,xPolicy);
}
- XXPolicyRefUser xPolUser = rangerAuditFields.populateAuditFields(new XXPolicyRefUser(), xPolicy);
-
- xPolUser.setPolicyId(policy.getId());
- xPolUser.setUserId(userId);
- xPolUser.setUserName(user);
-
- xPolUsers.add(xPolUser);
}
- RangerBizUtil.setBulkMode(oldBulkMode);
- daoMgr.getXXPolicyRefUser().batchCreate(xPolUsers);
List<XXPolicyRefAccessType> xPolAccesses = new ArrayList<>();
for (String accessType : accessTypes) {
@@ -290,18 +281,44 @@ public class PolicyRefUpdater {
daoMgr.getXXPolicyRefDataMaskType().batchCreate(xxDataMaskInfos);
}
- private Long createUserForPolicy(String user) {
+ private void createUserForPolicy(String user, XXPolicy xPolicy) {
LOG.warn("User specified in policy does not exist in ranger admin, creating new user, User = " + user);
- return xUserMgr.createExternalUser(user).getId();
+ final PolicyUserCreateContext policyUserCreateContext = new PolicyUserCreateContext(user, xPolicy);
+ Runnable CreateAndAssociateUser = new Runnable () {
+ @Override
+ public void run() {
+ Runnable realTask = new Runnable () {
+ @Override
+ public void run() {
+ doCreateAndAssociatePolicyUser(policyUserCreateContext);
+ }
+ };
+ transactionService.scheduleToExecuteInOwnTransaction(realTask, 0L);
+ }
+ };
+ rangerTransactionSynchronizationAdapter.executeOnTransactionCommit(CreateAndAssociateUser);
}
- private Long createGroupForPolicy(String group) {
+ private void createGroupForPolicy(String group, XXPolicy xPolicy) {
LOG.warn("Group specified in policy does not exist in ranger admin, creating new group, Group = " + group);
VXGroup vxGroup = new VXGroup();
vxGroup.setName(group);
vxGroup.setGroupSource(RangerCommonEnums.GROUP_EXTERNAL);
- VXGroup vxGroupCreated= xUserMgr.createXGroup(vxGroup);
- return vxGroupCreated.getId();
+ final PolicyGroupCreateContext policyGroupCreateContext = new PolicyGroupCreateContext(vxGroup, xPolicy);
+ Runnable createAndAssociatePolicyGroup = new Runnable() {
+ @Override
+ public void run() {
+ Runnable realTask = new Runnable() {
+ @Override
+ public void run() {
+ doCreateAndAssociatePolicyGroup(policyGroupCreateContext);
+ }
+ };
+ transactionService.scheduleToExecuteInOwnTransaction(realTask, 0L);
+ }
+ };
+ rangerTransactionSynchronizationAdapter.executeOnTransactionCommit(createAndAssociatePolicyGroup);
+
}
private Long createRoleForPolicy(String role) throws Exception {
@@ -363,4 +380,179 @@ public class PolicyRefUpdater {
return ret;
}
+
+ public void groupPolicyAssociation(XXPolicy xPolicy, Long groupId, String groupName) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.groupPolicyAssociation()");
+ }
+
+ XXPolicyRefGroup xPolGroup = rangerAuditFields.populateAuditFields(new XXPolicyRefGroup(), xPolicy);
+
+ xPolGroup.setPolicyId(xPolicy.getId());
+ xPolGroup.setGroupId(groupId);
+ xPolGroup.setGroupName(groupName);
+ daoMgr.getXXPolicyRefGroup().create(xPolGroup);
+ }
+
+ private static final class PolicyGroupCreateContext {
+ final VXGroup group;
+ final XXPolicy xPolicy;
+
+ PolicyGroupCreateContext(VXGroup group, XXPolicy xPolicy) {
+ this.group = group;
+ this.xPolicy = xPolicy;
+ }
+
+ @Override
+ public String toString() {
+ return "{group=" + group + ", xPolicy=" + xPolicy + "}";
+ }
+ }
+
+ void doAssociatePolicyGroup(final PolicyGroupCreateContext context) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.doAssociatePolicyGroup()");
+ }
+ XXGroup xGroup = daoMgr.getXXGroup().findByGroupName(context.group.getName());
+
+ if (xGroup == null) {
+ LOG.error("No Group created!! Irrecoverable error! PolicyGroupContext:[" + context + "]");
+ } else {
+ try {
+ groupPolicyAssociation(context.xPolicy, xGroup.getId(), context.group.getName());
+ } catch (Exception exception) {
+ LOG.error("Failed to associate group and policy, PolicyGroupContext:[" + context + "]", exception);
+ }
+ }
+ }
+
+ void doCreateAndAssociatePolicyGroup(final PolicyGroupCreateContext context) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.doCreateAndAssociatePolicyGroup()");
+ }
+ XXGroup xGroup = daoMgr.getXXGroup().findByGroupName(context.group.getName());
+
+ if (xGroup != null) {
+ groupPolicyAssociation(context.xPolicy, xGroup.getId(), context.group.getName());
+ } else {
+ try {
+ // Create group
+ VXGroup vXGroup = xGroupService.createXGroupWithOutLogin(context.group);
+ if (null != vXGroup) {
+ List<XXTrxLog> trxLogList = xGroupService.getTransactionLog(vXGroup, "create");
+ for(XXTrxLog xTrxLog : trxLogList) {
+ xTrxLog.setAddedByUserId(context.xPolicy.getAddedByUserId());
+ xTrxLog.setUpdatedByUserId(context.xPolicy.getAddedByUserId());
+ }
+ rangerBizUtil.createTrxLog(trxLogList);
+ }
+ } catch (Exception exception) {
+ LOG.error("Failed to create Group or to associate group and policy, PolicyGroupContext:[" + context + "]",
+ exception);
+ } finally {
+ // This transaction may still fail at commit time because another transaction
+ // has already created the group
+ // So, associate the group to policy in a different transaction
+ Runnable associatePolicyGroup = new Runnable() {
+ @Override
+ public void run() {
+ Runnable realTask = new Runnable() {
+ @Override
+ public void run() {
+ doAssociatePolicyGroup(context);
+ }
+ };
+ transactionService.scheduleToExecuteInOwnTransaction(realTask, 0L);
+ }
+ };
+ rangerTransactionSynchronizationAdapter.executeOnTransactionCompletion(associatePolicyGroup);
+
+ }
+ }
+ }
+
+ private static final class PolicyUserCreateContext {
+ final String userName;
+ final XXPolicy xPolicy;
+
+ PolicyUserCreateContext(String userName, XXPolicy xPolicy) {
+ this.userName = userName;
+ this.xPolicy = xPolicy;
+ }
+
+ @Override
+ public String toString() {
+ return "{userName=" + userName + ", xPolicy=" + xPolicy + "}";
+ }
+ }
+
+ public void userPolicyAssociation(XXPolicy xPolicy, Long userId, String userName) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.userPolicyAssociation()");
+ }
+
+ XXPolicyRefUser xPolUser = rangerAuditFields.populateAuditFields(new XXPolicyRefUser(), xPolicy);
+
+ xPolUser.setPolicyId(xPolicy.getId());
+ xPolUser.setUserId(userId);
+ xPolUser.setUserName(userName);
+ daoMgr.getXXPolicyRefUser().create(xPolUser);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<=== PolicyRefUpdater.userPolicyAssociation()");
+ }
+ }
+
+ void doAssociatePolicyUser(final PolicyUserCreateContext context) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.doAssociatePolicyUser()");
+ }
+ XXUser xUser = daoMgr.getXXUser().findByUserName(context.userName);
+
+ if (xUser == null) {
+ LOG.error("No User created!! Irrecoverable error! PolicyUserContext:[" + context + "]");
+ } else {
+ try {
+ userPolicyAssociation(context.xPolicy, xUser.getId(), context.userName);
+ } catch (Exception exception) {
+ LOG.error("Failed to associate user and policy, PolicyUserContext:[" + context + "]", exception);
+ }
+ }
+ }
+
+ void doCreateAndAssociatePolicyUser(final PolicyUserCreateContext context) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("===> PolicyRefUpdater.doCreateAndAssociatePolicyUser()");
+ }
+ XXUser xUser = daoMgr.getXXUser().findByUserName(context.userName);
+
+ if (xUser != null) {
+ userPolicyAssociation(context.xPolicy, xUser.getId(), context.userName);
+ } else {
+ try {
+ // Create External user
+ xUserMgr.createServiceConfigUser(context.userName);
+ } catch (Exception exception) {
+ LOG.error("Failed to create User or to associate user and policy, PolicyUserContext:[" + context + "]",
+ exception);
+ } finally {
+ // This transaction may still fail at commit time because another transaction
+ // has already created the user
+ // So, associate the user to policy in a different transaction
+ Runnable associatePolicyUser = new Runnable() {
+ @Override
+ public void run() {
+ Runnable realTask = new Runnable() {
+ @Override
+ public void run() {
+ doAssociatePolicyUser(context);
+ }
+ };
+ transactionService.scheduleToExecuteInOwnTransaction(realTask, 0L);
+ }
+ };
+ rangerTransactionSynchronizationAdapter.executeOnTransactionCompletion(associatePolicyUser);
+ }
+ }
+
+ }
}