You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pr...@apache.org on 2026/08/13 11:05:53 UTC
(ranger) 02/02: RANGER-5743: Docker setup for Apache Ranger ElasticSearch plugin and Integrate ES plugin with x-pack-security on ES 7.17
This is an automated email from the ASF dual-hosted git repository.
pradeepagrawal8184 pushed a commit to branch RANGER-5740_esplugin
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit 1aa4f4ea6d4cf14a07b951b46d0d0a3440e9c251
Author: Pradeep Agrawal <pa...@cloudera.com>
AuthorDate: Thu Aug 13 16:05:47 2026 +0530
RANGER-5743: Docker setup for Apache Ranger ElasticSearch plugin and Integrate ES plugin with x-pack-security on ES 7.17
---
.../audit/provider/AuditProviderFactory.java | 28 ++-
.../hadoop/config/RangerPluginConfig.java | 14 +-
.../org/apache/ranger/audit/rest/AuditREST.java | 22 +-
.../resources/conf/ranger-audit-ingestor-site.xml | 7 +
dev-support/ranger-docker/.dockerignore | 1 +
dev-support/ranger-docker/.env | 7 +
.../ranger-docker/Dockerfile.ranger-elasticsearch | 51 +++++
dev-support/ranger-docker/README.md | 37 ++-
.../docker-compose.ranger-elasticsearch.yml | 58 +++++
dev-support/ranger-docker/download-archives.sh | 9 +-
.../scripts/admin/create-ranger-services.py | 17 +-
.../scripts/elasticsearch/elasticsearch.yml | 27 +++
.../elasticsearch/patch-ranger-audit-xml.py | 69 ++++++
.../elasticsearch/patch-ranger-security-xml.py | 71 ++++++
.../ranger-elasticsearch-plugin-install.properties | 102 +++++++++
.../ranger-elasticsearch-post-setup.sh | 173 ++++++++++++++
.../elasticsearch/ranger-elasticsearch-setup.sh | 97 ++++++++
.../scripts/elasticsearch/ranger-elasticsearch.sh | 78 +++++++
.../scripts/kafka/ranger-kafka-setup.sh | 6 +-
.../ranger-docker/scripts/kdc/entrypoint.sh | 4 +-
distro/pom.xml | 24 ++
distro/src/main/assembly/plugin-elasticsearch.xml | 107 +++++----
distro/src/main/assembly/plugin-solr.xml | 8 +
.../conf/ranger-elasticsearch-audit-changes.cfg | 1 +
plugin-elasticsearch/pom.xml | 17 ++
plugin-elasticsearch/scripts/install.properties | 1 +
.../ElasticsearchAuditIngestorClient.java | 250 +++++++++++++++++++++
.../RangerElasticsearchAuditHandler.java | 32 ++-
.../authorizer/RangerElasticsearchAuthorizer.java | 30 ++-
.../elasticsearch/RangerServiceElasticsearch.java | 2 +-
pom.xml | 1 +
.../conf/plugin-descriptor.properties | 5 +-
.../conf/plugin-security.policy | 49 ++--
.../authorizer/RangerElasticsearchAuthorizer.java | 102 ---------
.../RangerElasticsearchAuthorizerDelegate.java | 94 ++++++++
.../plugin/RangerElasticsearchPlugin.java | 71 +++---
.../action/filter/RangerSecurityActionFilter.java | 36 ++-
ranger_in_docker | 2 +-
38 files changed, 1450 insertions(+), 260 deletions(-)
diff --git a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index ec42d89f1..16c012f9b 100644
--- a/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/core/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.ranger.audit.destination.AuditDestination;
+import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.queue.AuditAsyncQueue;
import org.apache.ranger.audit.queue.AuditBatchQueue;
import org.apache.ranger.audit.queue.AuditFileQueue;
@@ -77,7 +78,8 @@ public class AuditProviderFactory {
private String componentAppType = "";
private boolean mInitDone;
private JVMShutdownHook jvmShutdownHook;
- private final ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional"));
+ private final List<AuditHandler> requestThreadDestinations = new ArrayList<>();
+ private final ArrayList<String> hbaseAppTypes = new ArrayList<>(Arrays.asList("hbaseMaster", "hbaseRegional", "elasticsearch"));
public AuditProviderFactory() {
LOG.info("AuditProviderFactory: creating..");
@@ -107,6 +109,27 @@ public AuditHandler getAuditProvider() {
return mProvider;
}
+ /**
+ * Delivers an audit event directly to configured destinations on the calling thread,
+ * bypassing async/batch queue threads. Used by the Elasticsearch plugin where ES Security
+ * Manager grants network permissions only to the request thread.
+ */
+ public boolean logOnRequestThread(AuditEventBase event) {
+ if (event == null || requestThreadDestinations.isEmpty()) {
+ return false;
+ }
+
+ boolean ret = true;
+
+ for (AuditHandler handler : requestThreadDestinations) {
+ if (!handler.log(event)) {
+ ret = false;
+ }
+ }
+
+ return ret;
+ }
+
public boolean isInitDone() {
return mInitDone;
}
@@ -128,6 +151,8 @@ public synchronized void init(Properties props, String appType) {
LOG.warn("AuditProviderFactory.init(): already initialized! Will try to re-initialize");
}
+ requestThreadDestinations.clear();
+
mInitDone = true;
componentAppType = appType;
@@ -187,6 +212,7 @@ public synchronized void init(Properties props, String appType) {
if (destProvider != null) {
destProvider.init(props, destPropPrefix);
+ requestThreadDestinations.add(destProvider);
String queueName = MiscUtil.getStringProperty(props, destPropPrefix + "." + AuditQueue.PROP_QUEUE);
diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java
index c5ee9e54d..0e1a57f7f 100644
--- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java
+++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java
@@ -72,13 +72,6 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId,
addResourcesForServiceType(serviceType);
- this.serviceType = serviceType;
- this.appId = StringUtils.isEmpty(appId) ? serviceType : appId;
- this.propertyPrefix = "ranger.plugin." + serviceType;
- this.serviceName = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName;
-
- addResourcesForServiceName(this.serviceType, this.serviceName);
-
if (additionalConfigFiles != null) {
for (File configFile : additionalConfigFiles) {
try {
@@ -89,6 +82,13 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId,
}
}
+ this.serviceType = serviceType;
+ this.appId = StringUtils.isEmpty(appId) ? serviceType : appId;
+ this.propertyPrefix = "ranger.plugin." + serviceType;
+ this.serviceName = StringUtils.isEmpty(serviceName) ? this.get(propertyPrefix + ".service.name") : serviceName;
+
+ addResourcesForServiceName(this.serviceType, this.serviceName);
+
String trustedProxyAddressString = this.get(propertyPrefix + ".trusted.proxy.ipaddresses");
if (StringUtil.isEmpty(clusterName)) {
diff --git a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java
index 2ae7d14e0..ec830a91a 100644
--- a/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java
+++ b/audit-server/audit-ingestor/src/main/java/org/apache/ranger/audit/rest/AuditREST.java
@@ -58,13 +58,29 @@
public class AuditREST {
private static final Logger LOG = LoggerFactory.getLogger(AuditREST.class);
- private static final Map<String, Set<String>> allowedServiceUsers;
+ private static volatile Map<String, Set<String>> allowedServiceUsers;
static {
- allowedServiceUsers = initializeAllowedUsers();
initializeAuthToLocal();
}
+ private static Map<String, Set<String>> getAllowedServiceUsers() {
+ Map<String, Set<String>> ret = allowedServiceUsers;
+
+ if (ret == null) {
+ synchronized (AuditREST.class) {
+ ret = allowedServiceUsers;
+
+ if (ret == null) {
+ allowedServiceUsers = initializeAllowedUsers();
+ ret = allowedServiceUsers;
+ }
+ }
+ }
+
+ return ret;
+ }
+
@Autowired
AuditDestinationMgr auditDestinationMgr;
@@ -330,7 +346,7 @@ private boolean isAllowedServiceUser(String serviceName, String userName) {
boolean ret;
if (StringUtils.isNotBlank(serviceName) && StringUtils.isNotBlank(userName)) {
- Set<String> allowedUsers = allowedServiceUsers.get(serviceName);
+ Set<String> allowedUsers = getAllowedServiceUsers().get(serviceName);
ret = allowedUsers != null && allowedUsers.contains(userName);
} else {
diff --git a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml
index 1688b4dc4..d46528c49 100644
--- a/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml
+++ b/audit-server/audit-ingestor/src/main/resources/conf/ranger-audit-ingestor-site.xml
@@ -242,6 +242,12 @@
<description>Allowed users for dev_solr (Solr plugin)</description>
</property>
+ <property>
+ <name>ranger.audit.ingestor.service.dev_elasticsearch.allowed.users</name>
+ <value>elasticsearch</value>
+ <description>Allowed users for dev_elasticsearch (Elasticsearch plugin)</description>
+ </property>
+
<!-- AUTH_TO_LOCAL RULES FOR KERBEROS PRINCIPAL MAPPING -->
<property>
<name>ranger.audit.ingestor.auth.to.local</name>
@@ -249,6 +255,7 @@
RULE:[2:$1/$2@$0]([ndj]n/.*@.*|hdfs/.*@.*)s/.*/hdfs/
RULE:[2:$1/$2@$0]([rn]m/.*@.*|yarn/.*@.*)s/.*/yarn/
RULE:[2:$1/$2@$0](jhs/.*@.*)s/.*/mapred/
+ RULE:[2:$1/$2@$0](elasticsearch/.*@.*)s/.*/elasticsearch/
RULE:[1:$1@$0](.*@.*)s/@.*//
DEFAULT
</value>
diff --git a/dev-support/ranger-docker/.dockerignore b/dev-support/ranger-docker/.dockerignore
index 05026e91e..06204248b 100644
--- a/dev-support/ranger-docker/.dockerignore
+++ b/dev-support/ranger-docker/.dockerignore
@@ -17,5 +17,6 @@
!dist/ranger-*-trino-plugin.tar.gz
!dist/ranger-*-ozone-plugin.tar.gz
!dist/ranger-*-solr-plugin.tar.gz
+!dist/ranger-*-elasticsearch-plugin.tar.gz
!downloads/*
!scripts/*
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
index 9ec9bdf7c..446d79a8a 100644
--- a/dev-support/ranger-docker/.env
+++ b/dev-support/ranger-docker/.env
@@ -88,6 +88,13 @@ TRINO_VERSION=latest
# Open Search
OPENSEARCH_VERSION=1.3.19
+# Elasticsearch Configuration (authorization plugin testing)
+ELASTICSEARCH_VERSION=7.17.29
+ELASTICSEARCH_PLUGIN_VERSION=3.0.0-SNAPSHOT
+ELASTICSEARCH_BOOTSTRAP_PASSWORD=rangerR0cks!
+RANGER_ADMIN_USER=admin
+RANGER_ADMIN_PASSWORD=rangerR0cks!
+
# Debug Configuration
DEBUG_ADMIN=false
DEBUG_USERSYNC=false
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch b/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch
new file mode 100644
index 000000000..da42a46a3
--- /dev/null
+++ b/dev-support/ranger-docker/Dockerfile.ranger-elasticsearch
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG RANGER_BASE_IMAGE=apache/ranger-base
+ARG RANGER_BASE_VERSION=20260123-2-8
+FROM ${RANGER_BASE_IMAGE}:${RANGER_BASE_VERSION}
+
+ARG ELASTICSEARCH_VERSION
+ARG ELASTICSEARCH_PLUGIN_VERSION
+
+COPY ./dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz /home/ranger/dist/
+COPY ./downloads/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz /home/ranger/dist/
+COPY ./scripts/elasticsearch/*.sh ${RANGER_SCRIPTS}/
+COPY ./scripts/elasticsearch/patch-ranger-security-xml.py ${RANGER_SCRIPTS}/
+COPY ./scripts/elasticsearch/patch-ranger-audit-xml.py ${RANGER_SCRIPTS}/
+COPY ./downloads/jaxb-api-2.2.11.jar ${RANGER_SCRIPTS}/elasticsearch-lib/
+COPY ./downloads/jaxb-runtime-2.3.2.jar ${RANGER_SCRIPTS}/elasticsearch-lib/
+COPY ./downloads/javax.activation-api-1.2.0.jar ${RANGER_SCRIPTS}/elasticsearch-lib/
+
+RUN groupadd -r elasticsearch 2>/dev/null || true && \
+ useradd -r -g hadoop -d /opt/elasticsearch -s /bin/bash elasticsearch 2>/dev/null || true && \
+ tar xvfz /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz --directory=/opt/ && \
+ ln -s /opt/elasticsearch-${ELASTICSEARCH_VERSION} /opt/elasticsearch && \
+ rm -f /home/ranger/dist/elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz && \
+ mkdir -p /opt/elasticsearch/data /opt/elasticsearch/logs && \
+ tar xvfz /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz --directory=/opt/ranger && \
+ ln -s /opt/ranger/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin /opt/ranger/ranger-elasticsearch-plugin && \
+ rm -f /home/ranger/dist/ranger-${ELASTICSEARCH_PLUGIN_VERSION}-elasticsearch-plugin.tar.gz && \
+ rm -f /opt/ranger/ranger-elasticsearch-plugin/install.properties && \
+ chown -R elasticsearch:hadoop /opt/elasticsearch* && \
+ chmod 744 ${RANGER_SCRIPTS}/ranger-elasticsearch-setup.sh \
+ ${RANGER_SCRIPTS}/ranger-elasticsearch-post-setup.sh \
+ ${RANGER_SCRIPTS}/ranger-elasticsearch.sh
+
+ENV ELASTICSEARCH_HOME=/opt/elasticsearch
+ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/elasticsearch/bin
+
+ENTRYPOINT [ "/home/ranger/scripts/ranger-elasticsearch.sh" ]
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 76c9968a3..bab29ecbe 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -34,7 +34,7 @@ Use Dockerfiles in this directory to create docker images and run them to build
~~~
chmod +x download-archives.sh
# use a subset of the below to download specific services
- ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch
+ ./download-archives.sh hadoop hive hbase kafka knox ozone opensearch elasticsearch
~~~
- Execute following commands to set environment variables to build Apache Ranger docker containers:
@@ -157,6 +157,41 @@ docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml up
docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-opensearch.yml up -d
~~~
+#### Bring up elasticsearch container (authorization plugin testing):
+~~~
+# Prerequisites: build Ranger artifacts and download the Elasticsearch archive
+mvn clean package -pl distro -am -DskipTests
+cp target/ranger-* dev-support/ranger-docker/dist/
+cd dev-support/ranger-docker
+./download-archives.sh elasticsearch
+
+export RANGER_DB_TYPE=postgres
+
+# Host port 9201 maps to container 9200 (avoids conflict with OpenSearch on 9200).
+# On Linux, ensure vm.max_map_count >= 262144 (e.g. sudo sysctl -w vm.max_map_count=262144).
+docker compose -f docker-compose.ranger.yml -f docker-compose.ranger-solr.yml \
+ -f docker-compose.ranger-elasticsearch.yml up -d --build
+~~~
+
+Elasticsearch starts with X-Pack Security enabled (native realm). Default credentials:
+
+- `elastic` / value of `ELASTICSEARCH_BOOTSTRAP_PASSWORD` in `.env` (default: `rangerR0cks!`)
+- `testuser_2` / same password (created automatically for authorization testing)
+
+Smoke tests after the container is healthy:
+
+~~~
+# Authenticated request (expect 200 or policy-based 403, not 401)
+curl -u elastic:rangerR0cks! http://localhost:9201/test-index/_search
+
+# Unauthenticated request (expect 401)
+curl http://localhost:9201/test-index/_search
+~~~
+
+Ranger Admin registers the `dev_elasticsearch` service automatically on first startup.
+The plugin polls policies from Ranger Admin; allow up to 30 seconds after startup for
+policy refresh before running authorization tests.
+
#### OpenSearch audit flow (replace Solr for access audits)
OpenSearch can replace Solr for **audit storage and UI queries**. Ranger Admin reads audits via
diff --git a/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml b/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml
new file mode 100644
index 000000000..3acf5fa9e
--- /dev/null
+++ b/dev-support/ranger-docker/docker-compose.ranger-elasticsearch.yml
@@ -0,0 +1,58 @@
+services:
+ ranger-elasticsearch:
+ build:
+ context: .
+ dockerfile: Dockerfile.ranger-elasticsearch
+ args:
+ - RANGER_BASE_IMAGE=${RANGER_BASE_IMAGE}
+ - RANGER_BASE_VERSION=${RANGER_BASE_VERSION}
+ - ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION}
+ - ELASTICSEARCH_PLUGIN_VERSION=${ELASTICSEARCH_PLUGIN_VERSION}
+ image: ranger-elasticsearch
+ container_name: ranger-elasticsearch
+ hostname: ranger-elasticsearch.rangernw
+ volumes:
+ - ./dist/version:/home/ranger/dist/version:ro
+ - ./dist/keytabs/ranger-elasticsearch:/etc/keytabs
+ - ./scripts/kdc/krb5.conf:/etc/krb5.conf:ro
+ - ./scripts/elasticsearch/elasticsearch.yml:/home/ranger/scripts/elasticsearch.yml:ro
+ - ./scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties:/opt/ranger/ranger-elasticsearch-plugin/install.properties
+ - ./scripts/elasticsearch/patch-ranger-audit-xml.py:/home/ranger/scripts/patch-ranger-audit-xml.py:ro
+ - ./scripts/elasticsearch/ranger-elasticsearch.sh:/home/ranger/scripts/ranger-elasticsearch.sh:ro
+ - elasticsearch-data:/opt/elasticsearch/data
+ - elasticsearch-logs:/opt/elasticsearch/logs
+ stdin_open: true
+ tty: true
+ networks:
+ - ranger
+ ports:
+ - "9201:9200"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ nofile:
+ soft: 65536
+ hard: 65536
+ depends_on:
+ ranger-kdc:
+ condition: service_started
+ ranger:
+ condition: service_started
+ environment:
+ - KERBEROS_ENABLED=${KERBEROS_ENABLED}
+ - KRB5_CONFIG=/etc/krb5.conf
+ - ELASTICSEARCH_VERSION
+ - ELASTICSEARCH_PLUGIN_VERSION
+ - ELASTICSEARCH_BOOTSTRAP_PASSWORD
+ - RANGER_ADMIN_USER
+ - RANGER_ADMIN_PASSWORD
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+
+volumes:
+ elasticsearch-data:
+ elasticsearch-logs:
+
+networks:
+ ranger:
+ name: rangernw
diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh
index b107cc50d..063007561 100755
--- a/dev-support/ranger-docker/download-archives.sh
+++ b/dev-support/ranger-docker/download-archives.sh
@@ -17,7 +17,7 @@
# limitations under the License.
#
-# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone archives to a local cache directory.
+# Downloads HDFS/Hive/HBase/Kafka/Knox/Ozone/OpenSearch/Elasticsearch archives to a local cache directory.
# The downloaded archives will be used while building docker images that run these services.
#
@@ -139,6 +139,7 @@ then
downloadIfNotPresent knox-${KNOX_VERSION}.tar.gz https://archive.apache.org/dist/knox/${KNOX_VERSION}
extractOzoneIfNeeded
downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}
+ downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch
else
for arg in "$@"; do
if [[ $arg == 'hadoop' ]]
@@ -164,6 +165,12 @@ else
elif [[ $arg == 'opensearch' ]]
then
downloadIfNotPresent opensearch-${OPENSEARCH_VERSION}-linux-x64.tar.gz https://artifacts.opensearch.org/releases/bundle/opensearch/${OPENSEARCH_VERSION}
+ elif [[ $arg == 'elasticsearch' ]]
+ then
+ downloadIfNotPresent elasticsearch-${ELASTICSEARCH_VERSION}-linux-x86_64.tar.gz https://artifacts.elastic.co/downloads/elasticsearch
+ downloadIfNotPresent jaxb-api-2.2.11.jar https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.2.11
+ downloadIfNotPresent jaxb-runtime-2.3.2.jar https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/2.3.2
+ downloadIfNotPresent javax.activation-api-1.2.0.jar https://repo1.maven.org/maven2/javax/activation/javax.activation-api/1.2.0
else
echo "Passed argument $arg is invalid!"
fi
diff --git a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
index 7e2eae0ee..1dc95c948 100644
--- a/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
+++ b/dev-support/ranger-docker/scripts/admin/create-ranger-services.py
@@ -148,11 +148,24 @@ def service_not_exists(service):
'ranger.plugin.super.users': 'solr',
'ranger.plugin.solr.policy.refresh.synchronous':'true'}})
-services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone, solr]
+elasticsearch = RangerService({'name': 'dev_elasticsearch', 'type': 'elasticsearch',
+ 'configs': {'username': 'elastic',
+ 'elasticsearch.url': 'http://ranger-elasticsearch.rangernw:9200',
+ 'policy.download.auth.users': 'elastic,admin,elasticsearch',
+ 'tag.download.auth.users': 'elastic,admin,elasticsearch',
+ 'userstore.download.auth.users': 'elastic,admin,elasticsearch',
+ 'setup.additional.default.policies': 'true',
+ 'default-policy.1.name': 'index: test-index',
+ 'default-policy.1.resource.index': 'test-index',
+ 'default-policy.1.policyItem.1.users': 'elastic,testuser_2',
+ 'default-policy.1.policyItem.1.accessTypes': 'read,all',
+ 'ranger.plugin.elasticsearch.policy.refresh.synchronous': 'true'}})
+
+services = [hdfs, yarn, hive, hbase, kafka, knox, kms, trino, ozone, solr, elasticsearch]
for service in services:
try:
if service_not_exists(service):
ranger_client.create_service(service)
print(f" {service.name} service created!")
except Exception as e:
- print(f"An exception occured: {e}")
+ print(f"An exception occured: {e}")
\ No newline at end of file
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml b/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml
new file mode 100644
index 000000000..826f5375e
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/elasticsearch.yml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cluster.name: ranger-elasticsearch-cluster
+node.name: ranger-elasticsearch.rangernw
+network.host: 0.0.0.0
+discovery.type: single-node
+
+path.data: /opt/elasticsearch/data
+path.logs: /opt/elasticsearch/logs
+
+xpack.security.enabled: true
+xpack.security.http.ssl.enabled: false
+xpack.security.transport.ssl.enabled: false
+xpack.ml.enabled: false
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py
new file mode 100644
index 000000000..bafad172b
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-audit-xml.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import re
+
+
+def set_property(text, name, value):
+ pattern = rf"(<name>{re.escape(name)}</name>\s*<value>)[^<]*(</value>)"
+ if re.search(pattern, text):
+ return re.sub(pattern, lambda match: f"{match.group(1)}{value}{match.group(2)}", text, count=1)
+
+ insertion = (
+ f"\t<property>\n"
+ f"\t\t<name>{name}</name>\n"
+ f"\t\t<value>{value}</value>\n"
+ f"\t</property>\n"
+ f"</configuration>"
+ )
+ return text.replace("</configuration>", insertion, 1)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("audit_xml")
+ parser.add_argument("--auditserver-url")
+ parser.add_argument("--spool-dir")
+ parser.add_argument("--kerberos-enabled", action="store_true")
+ args = parser.parse_args()
+
+ with open(args.audit_xml, encoding="utf-8") as handle:
+ text = handle.read()
+
+ text = set_property(text, "xasecure.audit.is.enabled", "true")
+ text = set_property(text, "xasecure.audit.destination.auditserver", "true")
+
+ if args.auditserver_url:
+ text = set_property(text, "xasecure.audit.destination.auditserver.url", args.auditserver_url)
+
+ if args.spool_dir:
+ text = set_property(
+ text,
+ "xasecure.audit.destination.auditserver.batch.filespool.dir",
+ args.spool_dir,
+ )
+
+ if args.kerberos_enabled:
+ text = set_property(text, "xasecure.audit.destination.auditserver.authn.type", "kerberos")
+
+ with open(args.audit_xml, "w", encoding="utf-8") as handle:
+ handle.write(text)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py
new file mode 100644
index 000000000..dfd85e1f5
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/patch-ranger-security-xml.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import re
+import sys
+
+
+def set_property(text, name, value):
+ pattern = rf"(<name>{re.escape(name)}</name>\s*<value>)[^<]*(</value>)"
+ if re.search(pattern, text):
+ return re.sub(pattern, lambda match: f"{match.group(1)}{value}{match.group(2)}", text, count=1)
+
+ insertion = (
+ f"\t<property>\n"
+ f"\t\t<name>{name}</name>\n"
+ f"\t\t<value>{value}</value>\n"
+ f"\t</property>\n"
+ f"</configuration>"
+ )
+ return text.replace("</configuration>", insertion, 1)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("security_xml")
+ parser.add_argument("--cache-dir")
+ parser.add_argument("--poll-interval-ms")
+ parser.add_argument("--admin-user")
+ parser.add_argument("--admin-password")
+ parser.add_argument("--clear-admin-creds", action="store_true")
+ args = parser.parse_args()
+
+ with open(args.security_xml, encoding="utf-8") as handle:
+ text = handle.read()
+
+ if args.cache_dir:
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.cache.dir", args.cache_dir)
+
+ if args.poll_interval_ms:
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.pollIntervalMs", args.poll_interval_ms)
+
+ if args.clear_admin_creds:
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.username", "")
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.password", "")
+ else:
+ if args.admin_user is not None:
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.username", args.admin_user)
+ if args.admin_password is not None:
+ text = set_property(text, "ranger.plugin.elasticsearch.policy.rest.client.password", args.admin_password)
+
+ with open(args.security_xml, "w", encoding="utf-8") as handle:
+ handle.write(text)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties
new file mode 100644
index 000000000..82e94f7c6
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-plugin-install.properties
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+POLICY_MGR_URL=http://ranger.rangernw:6080
+REPOSITORY_NAME=dev_elasticsearch
+COMPONENT_INSTALL_DIR_NAME=/opt/elasticsearch
+POLICY_CACHE_FILE_PATH=/opt/elasticsearch/data/ranger-policycache
+RANGER_ADMIN_USER=admin
+RANGER_ADMIN_PASSWORD=rangerR0cks!
+
+CUSTOM_USER=elasticsearch
+CUSTOM_GROUP=hadoop
+
+XAAUDIT.SUMMARY.ENABLE=true
+
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://ranger-solr.rangernw:8983/solr/ranger_audits
+
+# Following properties are needed to get past installation script! Please don't remove
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit
+XAAUDIT.HDFS.DESTINTATION_FILE=hadoop
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/elasticsearch/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/elasticsearch/audit/archive
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+XAAUDIT.SOLR.ENABLE=false
+XAAUDIT.SOLR.URL=NONE
+XAAUDIT.SOLR.USER=NONE
+XAAUDIT.SOLR.PASSWORD=NONE
+XAAUDIT.SOLR.ZOOKEEPER=NONE
+XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/elasticsearch/audit/solr/spool
+XAAUDIT.SOLR.USE_INMEMORY_JAAS_CFG=false
+
+XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_NAME=com.sun.security.auth.module.Krb5LoginModule
+XAAUDIT.JAAS.CLIENT.LOGIN_MODULE_CONTROL_FLAG=required
+XAAUDIT.JAAS.CLIENT.OPTION.USE_KEY_TAB=true
+XAAUDIT.JAAS.CLIENT.OPTION.STORE_KEY=true
+XAAUDIT.JAAS.CLIENT.OPTION.USE_TICKET_CACHE=true
+XAAUDIT.JAAS.CLIENT.OPTION.SERVICE_NAME=elasticsearch
+XAAUDIT.JAAS.CLIENT.OPTION.KEY_TAB=/etc/keytabs/elasticsearch.keytab
+XAAUDIT.JAAS.CLIENT.OPTION.PRINCIPAL=elasticsearch/ranger-elasticsearch.rangernw@EXAMPLE.COM
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.HDFS.ENABLE=false
+XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop.rangernw:9000/ranger/audit
+XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/elasticsearch/audit/hdfs/spool
+
+XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
+XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
+XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
+
+XAAUDIT.LOG4J.ENABLE=false
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=false
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+XAAUDIT.AUDITSERVER.ENABLE=true
+XAAUDIT.AUDITSERVER.URL=http://ranger-audit-ingestor.rangernw:7081
+XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/opt/elasticsearch/data/ranger-audit-spool
+XAAUDIT.AUDITSERVER.AUTHN.TYPE=kerberos
+
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh
new file mode 100755
index 000000000..2631014b9
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-post-setup.sh
@@ -0,0 +1,173 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+ES_URL="http://127.0.0.1:9200"
+ES_USER="elastic"
+ES_PASS="${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}"
+RANGER_URL="http://ranger.rangernw:6080"
+RANGER_USER="${RANGER_ADMIN_USER:-admin}"
+RANGER_PASS="${RANGER_ADMIN_PASSWORD:-rangerR0cks!}"
+SERVICE_NAME="dev_elasticsearch"
+CACHE_DIR="${ELASTICSEARCH_HOME}/data/ranger-policycache"
+CACHE_FILE="${CACHE_DIR}/elasticsearch_${SERVICE_NAME}.json"
+MAX_ATTEMPTS=60
+MODE="${1:-all}"
+
+wait_for_ranger_admin() {
+ local attempt=0
+ until curl -s --max-time 5 -u "${RANGER_USER}:${RANGER_PASS}" \
+ "${RANGER_URL}/service/public/v2/api/version" >/dev/null 2>&1; do
+ attempt=$((attempt + 1))
+ if [ "${attempt}" -ge "${MAX_ATTEMPTS}" ]; then
+ echo "ERROR: Ranger Admin did not become reachable in time." >&2
+ exit 1
+ fi
+ sleep 5
+ done
+}
+
+wait_for_authorized_search() {
+ local attempt=0
+ local http_code=""
+ until http_code="$(curl -s --max-time 10 -o /dev/null -w '%{http_code}' -u "${ES_USER}:${ES_PASS}" \
+ "${ES_URL}/test-index/_search")" && echo "${http_code}" | grep -qE '^(200|404)$'; do
+ attempt=$((attempt + 1))
+ if [ "${attempt}" -ge "${MAX_ATTEMPTS}" ]; then
+ echo "ERROR: Elasticsearch did not authorize ${ES_USER} in time." >&2
+ exit 1
+ fi
+ sleep 5
+ done
+}
+
+seed_policy_cache() {
+ echo "Seeding Ranger policy cache under ${CACHE_DIR}..."
+ mkdir -p "${CACHE_DIR}"
+ python3 - "${RANGER_URL}" "${RANGER_USER}" "${RANGER_PASS}" "${SERVICE_NAME}" "${CACHE_FILE}" <<'PY'
+import json
+import sys
+import urllib.request
+import base64
+
+ranger_url, user, password, service_name, cache_file = sys.argv[1:6]
+auth = base64.b64encode(f"{user}:{password}".encode()).decode()
+
+def fetch(path):
+ req = urllib.request.Request(f"{ranger_url}{path}")
+ req.add_header("Authorization", f"Basic {auth}")
+ with urllib.request.urlopen(req) as resp:
+ return json.load(resp)
+
+service = fetch(f"/service/public/v2/api/service/name/{service_name}")
+policies = fetch(f"/service/public/v2/api/service/{service_name}/policy")
+if not isinstance(policies, list):
+ raise SystemExit(f"unexpected policy payload type: {type(policies)}")
+
+for policy in policies:
+ resources = policy.get("resources") or {}
+ index_resource = resources.get("index") or {}
+ index_values = index_resource.get("values") or []
+ if "test-index" in index_values:
+ for item in policy.get("policyItems") or []:
+ item["users"] = ["elastic", "testuser_2"]
+ policy_id = policy.get("id")
+ if policy_id is not None:
+ update_body = {
+ "id": policy_id,
+ "name": policy.get("name"),
+ "service": policy.get("service", service_name),
+ "serviceType": policy.get("serviceType", "elasticsearch"),
+ "resources": policy.get("resources"),
+ "policyItems": policy.get("policyItems"),
+ "isEnabled": policy.get("isEnabled", True),
+ "policyType": policy.get("policyType", 0),
+ "isAuditEnabled": policy.get("isAuditEnabled", True),
+ }
+ try:
+ update_req = urllib.request.Request(
+ f"{ranger_url}/service/public/v2/api/policy/{policy_id}",
+ data=json.dumps(update_body).encode(),
+ method="PUT",
+ )
+ update_req.add_header("Authorization", f"Basic {auth}")
+ update_req.add_header("Content-Type", "application/json")
+ with urllib.request.urlopen(update_req) as resp:
+ if resp.status != 200:
+ print(f"WARN: Ranger Admin policy update returned HTTP {resp.status}", file=sys.stderr)
+ except Exception as exc:
+ print(f"WARN: Could not update test-index policy in Ranger Admin: {exc}", file=sys.stderr)
+
+policy_version = service.get("policyVersion")
+if policy_version is None:
+ policy_version = max((p.get("version") or 1) for p in policies) if policies else 1
+
+payload = {
+ "serviceName": service_name,
+ "serviceId": service.get("id", 1),
+ "policyVersion": policy_version,
+ "policies": policies,
+}
+
+with open(cache_file, "w", encoding="utf-8") as handle:
+ json.dump(payload, handle)
+PY
+
+ chown elasticsearch:hadoop "${CACHE_FILE}"
+ chmod 640 "${CACHE_FILE}"
+}
+
+disable_admin_policy_download() {
+ SECURITY_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-security.xml"
+ if [ -f "${SECURITY_XML}" ]; then
+ python3 "${RANGER_SCRIPTS}/patch-ranger-security-xml.py" "${SECURITY_XML}" --clear-admin-creds
+ fi
+}
+
+create_test_fixtures() {
+ echo "Creating test index test-index..."
+ curl -s --max-time 30 -u "${ES_USER}:${ES_PASS}" -X PUT "${ES_URL}/test-index" \
+ -H 'Content-Type: application/json' \
+ -d '{"settings":{"number_of_shards":1,"number_of_replicas":0}}' \
+ | grep -q '"acknowledged":true' || echo "test-index may already exist"
+}
+
+case "${MODE}" in
+ seed)
+ wait_for_ranger_admin
+ seed_policy_cache
+ ;;
+ fixtures)
+ sleep 30
+ wait_for_authorized_search
+ create_test_fixtures
+ ;;
+ all)
+ wait_for_ranger_admin
+ seed_policy_cache
+ sleep 30
+ wait_for_authorized_search
+ create_test_fixtures
+ ;;
+ *)
+ echo "Usage: $0 [seed|fixtures|all]" >&2
+ exit 1
+ ;;
+esac
+
+echo "Elasticsearch post-setup (${MODE}) completed successfully"
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh
new file mode 100755
index 000000000..91fd1660d
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch-setup.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+cp ${RANGER_SCRIPTS}/elasticsearch.yml ${ELASTICSEARCH_HOME}/config/elasticsearch.yml
+
+if [ ! -f "${ELASTICSEARCH_HOME}/config/elasticsearch.keystore" ]; then
+ su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-keystore create"
+ echo "${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}" | \
+ su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-keystore add -x bootstrap.password"
+fi
+
+chown -R elasticsearch:hadoop ${ELASTICSEARCH_HOME}
+
+# File-realm users for authorization smoke tests (offline; Ranger blocks security REST APIs).
+ES_BOOTSTRAP_PASSWORD="${ELASTICSEARCH_BOOTSTRAP_PASSWORD:-rangerR0cks!}"
+cat > "${ELASTICSEARCH_HOME}/config/roles.yml" <<'EOF'
+ranger_test_index_reader:
+ indices:
+ - names: ['test-index']
+ privileges: ['read', 'view_index_metadata']
+EOF
+chown elasticsearch:hadoop "${ELASTICSEARCH_HOME}/config/roles.yml"
+chmod 640 "${ELASTICSEARCH_HOME}/config/roles.yml"
+for smoke_user in testuser_2 testuser_denied; do
+ su -s /bin/bash elasticsearch -c "${ELASTICSEARCH_HOME}/bin/elasticsearch-users useradd ${smoke_user} -p \"${ES_BOOTSTRAP_PASSWORD}\" -r ranger_test_index_reader" \
+ 2>/dev/null || true
+done
+
+# ES plugin security manager allows read but not write under /etc/ranger; keep cache under ES data.
+export POLICY_CACHE_FILE_PATH="${ELASTICSEARCH_HOME}/data/ranger-policycache"
+mkdir -p "${POLICY_CACHE_FILE_PATH}"
+chown elasticsearch:hadoop "${POLICY_CACHE_FILE_PATH}"
+chmod 750 "${POLICY_CACHE_FILE_PATH}"
+
+AUDIT_SPOOL_DIR="${ELASTICSEARCH_HOME}/data/ranger-audit-spool"
+mkdir -p "${AUDIT_SPOOL_DIR}"
+chown elasticsearch:hadoop "${AUDIT_SPOOL_DIR}"
+chmod 750 "${AUDIT_SPOOL_DIR}"
+
+cd ${RANGER_HOME}/ranger-elasticsearch-plugin
+./enable-elasticsearch-plugin.sh
+
+# enable-agent.sh resets POLICY_CACHE_FILE_PATH to /etc/ranger/...; patch Ranger config for docker.
+SECURITY_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-security.xml"
+if [ -f "${SECURITY_XML}" ]; then
+ python3 "${RANGER_SCRIPTS}/patch-ranger-security-xml.py" "${SECURITY_XML}" \
+ --cache-dir "${POLICY_CACHE_FILE_PATH}" \
+ --poll-interval-ms "86400000" \
+ --admin-user "${RANGER_ADMIN_USER:-admin}" \
+ --admin-password "${RANGER_ADMIN_PASSWORD:-rangerR0cks!}"
+fi
+
+AUDIT_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-audit.xml"
+if [ -f "${AUDIT_XML}" ]; then
+ python3 "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" "${AUDIT_XML}" \
+ --auditserver-url "http://ranger-audit-ingestor.rangernw:7081" \
+ --spool-dir "${AUDIT_SPOOL_DIR}"
+fi
+
+# Replace plugin symlinks with local copies; ES security manager cannot read jars via /opt/ranger symlinks.
+RANGER_PLUGIN_LIB="${RANGER_HOME}/ranger-elasticsearch-plugin/lib/ranger-elasticsearch-plugin"
+PLUGIN_DIR="${ELASTICSEARCH_HOME}/plugins/ranger-elasticsearch-plugin"
+rm -rf "${PLUGIN_DIR}"
+mkdir -p "${PLUGIN_DIR}"
+cp -a "${RANGER_PLUGIN_LIB}/." "${PLUGIN_DIR}/"
+# ES 7.17+ does not allow plugins to create classloaders; load impl jars on the plugin classpath.
+if [ -d "${RANGER_PLUGIN_LIB}/ranger-elasticsearch-plugin-impl" ]; then
+ cp "${RANGER_PLUGIN_LIB}/ranger-elasticsearch-plugin-impl/"*.jar "${PLUGIN_DIR}/"
+fi
+# Shim runtime deps (exclude jars already provided by x-pack-security / x-pack-core / ES lib)
+for jar in commons-lang3 commons-collections hadoop-client-api hadoop-client-runtime commons-configuration gson jackson-databind jackson-annotations; do
+ cp "${RANGER_HOME}/ranger-elasticsearch-plugin/install/lib/${jar}-"*.jar "${PLUGIN_DIR}/" 2>/dev/null || true
+done
+if [ -d "${RANGER_SCRIPTS}/elasticsearch-lib" ]; then
+ cp "${RANGER_SCRIPTS}/elasticsearch-lib/"*.jar "${PLUGIN_DIR}/" 2>/dev/null || true
+fi
+cp "${RANGER_HOME}/ranger-elasticsearch-plugin/lib/ranger-elasticsearch-plugin/plugin-security.policy" "${PLUGIN_DIR}/" 2>/dev/null || true
+chown -R elasticsearch:hadoop "${PLUGIN_DIR}" "${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin"
+
+echo "Elasticsearch Ranger plugin setup completed successfully"
diff --git a/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh
new file mode 100755
index 000000000..9551dd5c6
--- /dev/null
+++ b/dev-support/ranger-docker/scripts/elasticsearch/ranger-elasticsearch.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ! -e ${ELASTICSEARCH_HOME}/.setupDone ]
+then
+ if "${RANGER_SCRIPTS}"/ranger-elasticsearch-setup.sh;
+ then
+ touch "${ELASTICSEARCH_HOME}"/.setupDone
+ else
+ echo "Ranger Elasticsearch Setup Script didn't complete proper execution." >&2
+ exit 1
+ fi
+fi
+
+AUDIT_SPOOL_DIR="${ELASTICSEARCH_HOME}/data/ranger-audit-spool"
+mkdir -p "${AUDIT_SPOOL_DIR}"
+chown elasticsearch:hadoop "${AUDIT_SPOOL_DIR}" 2>/dev/null || true
+chmod 750 "${AUDIT_SPOOL_DIR}" 2>/dev/null || true
+
+AUDIT_XML="${ELASTICSEARCH_HOME}/config/ranger-elasticsearch-plugin/ranger-elasticsearch-audit.xml"
+if [ -f "${AUDIT_XML}" ] && [ -f "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" ]; then
+ KERBEROS_PATCH_ARG=""
+ if [ "${KERBEROS_ENABLED}" = "true" ]; then
+ KERBEROS_PATCH_ARG="--kerberos-enabled"
+ fi
+ python3 "${RANGER_SCRIPTS}/patch-ranger-audit-xml.py" "${AUDIT_XML}" \
+ --auditserver-url "http://ranger-audit-ingestor.rangernw:7081" \
+ --spool-dir "${AUDIT_SPOOL_DIR}" \
+ ${KERBEROS_PATCH_ARG}
+ chown elasticsearch:hadoop "${AUDIT_XML}" 2>/dev/null || true
+fi
+
+if [ ! -e ${ELASTICSEARCH_HOME}/.postSetupDone ]
+then
+ if ! "${RANGER_SCRIPTS}"/ranger-elasticsearch-post-setup.sh seed;
+ then
+ echo "Ranger Elasticsearch Post-Setup Script didn't complete proper execution." >&2
+ exit 1
+ fi
+fi
+
+su -s /bin/bash elasticsearch -c "cd ${ELASTICSEARCH_HOME} && ES_JAVA_OPTS='${ES_JAVA_OPTS}' ./bin/elasticsearch" &
+ES_PID=$!
+
+if [ ! -e ${ELASTICSEARCH_HOME}/.postSetupDone ]
+then
+ if "${RANGER_SCRIPTS}"/ranger-elasticsearch-post-setup.sh fixtures;
+ then
+ touch "${ELASTICSEARCH_HOME}"/.postSetupDone
+ else
+ echo "Ranger Elasticsearch Post-Setup Script didn't complete proper execution." >&2
+ kill "${ES_PID}" 2>/dev/null || true
+ exit 1
+ fi
+fi
+
+if ! ps -p "${ES_PID}" > /dev/null 2>&1
+then
+ echo "The Elasticsearch process exited unexpectedly." >&2
+ exit 1
+fi
+
+tail --pid="${ES_PID}" -f /dev/null
diff --git a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
index 067042ada..ee212d49e 100755
--- a/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
+++ b/dev-support/ranger-docker/scripts/kafka/ranger-kafka-setup.sh
@@ -62,11 +62,7 @@ if [ "${KERBEROS_ENABLED}" == "true" ]; then
# Kerberos service name
sasl.kerberos.service.name=kafka
- # Ranger authorization
- authorizer.class.name=org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer
-
- # Super users bypass Ranger authorization for admin operations
- super.users=User:kafka
+ # Broker-only for audit pipeline; Ranger Kafka authorizer needs audit-core on broker classpath.
EOF
else
echo "Configuring Kafka with PLAINTEXT (no Kerberos)"
diff --git a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
index 4946cc026..39f800158 100644
--- a/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
+++ b/dev-support/ranger-docker/scripts/kdc/entrypoint.sh
@@ -121,7 +121,9 @@ function create_keytabs() {
create_principal_and_keytab opensearch ranger-opensearch
create_principal_and_keytab HTTP ranger-opensearch
-
+
+ create_principal_and_keytab elasticsearch ranger-elasticsearch
+
create_principal_and_keytab om om
create_principal_and_keytab scm scm
create_principal_and_keytab dn datanode
diff --git a/distro/pom.xml b/distro/pom.xml
index fa97c3fbd..272ecde74 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -29,6 +29,30 @@
<description>Apache Ranger Distribution</description>
<dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${elasticsearch.jackson.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${elasticsearch.jackson.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>${javax.annotation-api.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>audit-dispatcher-hdfs</artifactId>
diff --git a/distro/src/main/assembly/plugin-elasticsearch.xml b/distro/src/main/assembly/plugin-elasticsearch.xml
index 777437f24..75a4c9494 100644
--- a/distro/src/main/assembly/plugin-elasticsearch.xml
+++ b/distro/src/main/assembly/plugin-elasticsearch.xml
@@ -37,8 +37,9 @@
<fileMode>644</fileMode>
<includes>
<include>commons-collections:commons-collections</include>
- <include>org.slf4j:slf4j-api</include>
- <include>org.slf4j:slf4j-log4j12</include>
+ <include>org.apache.commons:commons-lang3</include>
+ <include>org.apache.httpcomponents:httpclient</include>
+ <include>org.apache.httpcomponents:httpcore</include>
</includes>
</binaries>
</moduleSet>
@@ -56,65 +57,40 @@
<include>org.apache.ranger:ranger-elasticsearch-plugin</include>
</includes>
<binaries>
- <outputDirectory>lib/ranger-elasticsearch-plugin/ranger-elasticsearch-plugin-impl</outputDirectory>
- <includeDependencies>false</includeDependencies>
+ <outputDirectory>lib/ranger-elasticsearch-plugin</outputDirectory>
+ <includeDependencies>true</includeDependencies>
<unpack>false</unpack>
<directoryMode>755</directoryMode>
<fileMode>644</fileMode>
<includes>
- <include>com.carrotsearch:hppc</include>
- <include>com.fasterxml.jackson.core:jackson-annotations:jar:${fasterxml.jackson.version}</include>
- <include>com.fasterxml.jackson.core:jackson-core:jar:${fasterxml.jackson.version}</include>
- <include>com.fasterxml.jackson.core:jackson-databind:${fasterxml.jackson.version}</include>
- <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include>
- <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include>
+ <include>org.apache.ranger:ranger-audit-core</include>
+ <include>org.apache.ranger:ranger-audit-dest-auditserver</include>
+ <include>org.apache.ranger:ranger-authz-api</include>
+ <include>org.apache.ranger:ranger-common-utils</include>
+ <include>org.apache.ranger:ranger-plugins-cred</include>
+ <include>org.apache.ranger:ranger-plugins-common</include>
+ <include>org.apache.ranger:ugsync-util</include>
+ <include>org.apache.ranger:ranger-elasticsearch-plugin</include>
<include>com.google.code.gson:gson</include>
- <include>commons-codec:commons-codec</include>
- <include>commons-collections:commons-collections</include>
- <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include>
- <include>commons-io:commons-io</include>
- <include>commons-logging:commons-logging:jar:${commons.logging.version}</include>
- <include>io.airlift:aircompressor:jar:${aircompressor.version}</include>
- <include>joda-time:joda-time</include>
- <include>org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version}</include>
- <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include>
- <include>org.apache.hadoop:hadoop-auth:jar:${hadoop.version}</include>
+ <include>commons-configuration:commons-configuration</include>
+ <include>javax.ws.rs:javax.ws.rs-api</include>
+ <include>javax.annotation:javax.annotation-api</include>
+ <include>org.glassfish.jersey.core:jersey-client</include>
+ <include>org.glassfish.jersey.core:jersey-common</include>
+ <include>org.glassfish.jersey.inject:jersey-hk2</include>
+ <include>org.glassfish.jersey.ext:jersey-entity-filtering</include>
+ <include>org.glassfish.jersey.media:jersey-media-json-jackson</include>
+ <include>org.glassfish.hk2:hk2-api</include>
+ <include>org.glassfish.hk2:hk2-locator</include>
+ <include>org.glassfish.hk2:hk2-utils</include>
+ <include>org.glassfish.hk2.external:aopalliance-repackaged</include>
+ <include>org.glassfish.hk2.external:jakarta.inject</include>
+ <include>org.javassist:javassist</include>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include>
+ <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include>
<include>org.apache.hadoop:hadoop-client-api:jar:${hadoop.version}</include>
<include>org.apache.hadoop:hadoop-client-runtime:jar:${hadoop.version}</include>
- <include>org.apache.hive:hive-storage-api:jar:${hive.storage-api.version}</include>
- <include>org.apache.httpcomponents:httpasyncclient:jar:${httpcomponents.httpasyncclient.version}</include>
- <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
- <include>org.apache.httpcomponents:httpcore-nio:jar:${httpcomponents.httpcore.version}</include>
- <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
- <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
- <include>org.apache.lucene:lucene-core</include>
- <include>org.apache.orc:orc-core:jar:${orc.version}</include>
- <include>org.apache.orc:orc-shims:jar:${orc.version}</include>
- <include>org.apache.solr:solr-solrj:jar:${solr.version}</include>
- <include>org.eclipse.jetty:jetty-client:jar:${jetty-client.version}</include>
- <include>org.elasticsearch.client:elasticsearch-rest-client</include>
- <include>org.elasticsearch.client:elasticsearch-rest-high-level-client</include>
- <include>org.elasticsearch.plugin:lang-mustache-client</include>
- <include>org.elasticsearch.plugin:rank-eval-client</include>
- <include>org.elasticsearch:elasticsearch-core</include>
- <include>org.elasticsearch:elasticsearch-x-content</include>
- <include>org.elasticsearch:elasticsearch</include>
- <include>org.noggit:noggit:jar:${noggit.version}</include>
- <!-- GraalJS 25.x runtime (js is now a POM, not a JAR) -->
- <include>org.graalvm.js:js-language:jar:${graalvm.version}</include>
- <include>org.graalvm.js:js-scriptengine:jar:${graalvm.version}</include>
- <include>org.graalvm.polyglot:polyglot:jar:${graalvm.version}</include>
- <include>org.graalvm.regex:regex:jar:${graalvm.version}</include>
- <include>org.graalvm.truffle:truffle-api:jar:${graalvm.version}</include>
- <include>org.graalvm.truffle:truffle-runtime:jar:${graalvm.version}</include>
- <include>org.graalvm.truffle:truffle-compiler:jar:${graalvm.version}</include>
- <include>org.graalvm.sdk:jniutils:jar:${graalvm.version}</include>
- <include>org.graalvm.sdk:collections:jar:${graalvm.version}</include>
- <include>org.graalvm.sdk:nativeimage:jar:${graalvm.version}</include>
- <include>org.graalvm.sdk:word:jar:${graalvm.version}</include>
- <include>org.graalvm.shadowed:icu4j:jar:${graalvm.version}</include>
- <include>org.graalvm.shadowed:xz:jar:${graalvm.version}</include>
-
</includes>
</binaries>
</moduleSet>
@@ -127,7 +103,7 @@
</includes>
<binaries>
<outputDirectory>install/lib</outputDirectory>
- <includeDependencies>false</includeDependencies>
+ <includeDependencies>true</includeDependencies>
<unpack>false</unpack>
<directoryMode>755</directoryMode>
<fileMode>644</fileMode>
@@ -219,4 +195,25 @@
<fileMode>755</fileMode>
</file>
</files>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib/ranger-elasticsearch-plugin</outputDirectory>
+ <scope>provided</scope>
+ <includes>
+ <include>com.fasterxml.jackson.core:jackson-annotations</include>
+ <include>com.fasterxml.jackson.core:jackson-databind</include>
+ <include>javax.annotation:javax.annotation-api</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib/ranger-elasticsearch-plugin</outputDirectory>
+ <includes>
+ <include>javax.xml.bind:jaxb-api:jar:2.2.11</include>
+ <include>org.glassfish.jaxb:jaxb-runtime:jar:2.3.2</include>
+ <include>javax.activation:javax.activation-api:jar:1.2.0</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
</assembly>
diff --git a/distro/src/main/assembly/plugin-solr.xml b/distro/src/main/assembly/plugin-solr.xml
index 6398f3aee..206cebc2f 100644
--- a/distro/src/main/assembly/plugin-solr.xml
+++ b/distro/src/main/assembly/plugin-solr.xml
@@ -52,6 +52,14 @@
<directoryMode>755</directoryMode>
<fileMode>644</fileMode>
<includes>
+ <include>org.apache.ranger:ranger-audit-core</include>
+ <include>org.apache.ranger:ranger-audit-dest-auditserver</include>
+ <include>org.apache.ranger:ranger-authz-api</include>
+ <include>org.apache.ranger:ranger-common-utils</include>
+ <include>org.apache.ranger:ranger-plugins-cred</include>
+ <include>org.apache.ranger:ranger-plugins-common</include>
+ <include>org.apache.ranger:ugsync-util</include>
+ <include>org.apache.ranger:ranger-solr-plugin</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:${fasterxml.jackson.version}</include>
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:${fasterxml.jackson.version}</include>
<include>com.google.code.gson:gson</include>
diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
index c81a52746..0c9466481 100644
--- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
+++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg
@@ -94,3 +94,4 @@ xasecure.audit.destination.log4j.logger %XAAUDIT.LOG4J.DESTINATIO
xasecure.audit.destination.auditserver %XAAUDIT.AUDITSERVER.ENABLE% mod create-if-not-exists
xasecure.audit.destination.auditserver.url %XAAUDIT.AUDITSERVER.URL% mod create-if-not-exists
xasecure.audit.destination.auditserver.batch.filespool.dir %XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR% mod create-if-not-exists
+xasecure.audit.destination.auditserver.authn.type %XAAUDIT.AUDITSERVER.AUTHN.TYPE% mod create-if-not-exists
diff --git a/plugin-elasticsearch/pom.xml b/plugin-elasticsearch/pom.xml
index 5f90340f1..b2b2cf85a 100644
--- a/plugin-elasticsearch/pom.xml
+++ b/plugin-elasticsearch/pom.xml
@@ -31,6 +31,23 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
+ <!-- ES 7.17 bundles jackson-core 2.14.2; ship databind/annotations only to avoid jar hell -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${elasticsearch.jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${elasticsearch.jackson.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties
index 35ee159ae..225486c4f 100644
--- a/plugin-elasticsearch/scripts/install.properties
+++ b/plugin-elasticsearch/scripts/install.properties
@@ -120,6 +120,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
XAAUDIT.AUDITSERVER.ENABLE=false
XAAUDIT.AUDITSERVER.URL=http://ranger-audit:7081
XAAUDIT.AUDITSERVER.FILE_SPOOL_DIR=/var/log/hive/audit/http/spool
+XAAUDIT.AUDITSERVER.AUTHN.TYPE=%EMPTY%
# End of V3 properties
#
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java
new file mode 100644
index 000000000..7ee943934
--- /dev/null
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/ElasticsearchAuditIngestorClient.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ranger.authorization.elasticsearch.authorizer;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.http.HttpStatus;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.elasticsearch.SpecialPermission;
+import org.ietf.jgss.GSSContext;
+import org.ietf.jgss.GSSManager;
+import org.ietf.jgss.GSSName;
+import org.ietf.jgss.Oid;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Posts access audits to the Ranger audit ingestor on the Elasticsearch request thread.
+ * Uses {@link HttpURLConnection} inside {@code SpecialPermission.check()} / {@code doPrivileged}
+ * so ES Security Manager allows outbound sockets (Jersey/async clients do not).
+ * When JAAS Kerberos settings are present in the audit configuration, posts use SPNEGO
+ * without {@code AuthenticatedURL} (ES plugin policy cannot grant cookie-handler permissions).
+ */
+final class ElasticsearchAuditIngestorClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchAuditIngestorClient.class);
+
+ private static final String REST_PATH_POST = "/api/audit/access";
+ private static final String QUERY_PARAM_SERVICE = "serviceName";
+ private static final String QUERY_PARAM_APP_ID = "appId";
+ private static final String AUTHZ_HEADER = "Authorization";
+ private static final String NEGOTIATE_PREFIX = "Negotiate ";
+
+ private static final String JAAS_PRINCIPAL_PROP = "xasecure.audit.jaas.Client.option.principal";
+ private static final String JAAS_KEYTAB_PROP = "xasecure.audit.jaas.Client.option.keyTab";
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final Object KERBEROS_INIT_LOCK = new Object();
+
+ private static volatile boolean kerberosInitialized;
+
+ private ElasticsearchAuditIngestorClient() {
+ }
+
+ static void init(Configuration config) {
+ if (config == null || kerberosInitialized || !isKerberosConfigured(config)) {
+ return;
+ }
+
+ synchronized (KERBEROS_INIT_LOCK) {
+ if (kerberosInitialized || !isKerberosConfigured(config)) {
+ return;
+ }
+
+ try {
+ SpecialPermission.check();
+
+ AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
+ initKerberos(config);
+
+ return null;
+ });
+
+ kerberosInitialized = true;
+ } catch (PrivilegedActionException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+
+ LOG.error("Failed to initialize Kerberos for audit ingestor client: {}", cause.getMessage(), cause);
+ } catch (RuntimeException e) {
+ LOG.error("Failed to initialize Kerberos for audit ingestor client: {}", e.getMessage(), e);
+ }
+ }
+ }
+
+ static boolean post(String auditServerBaseUrl, AuthzAuditEvent auditEvent) {
+ if (StringUtils.isBlank(auditServerBaseUrl) || auditEvent == null) {
+ return false;
+ }
+
+ try {
+ SpecialPermission.check();
+
+ return AccessController.doPrivileged((PrivilegedExceptionAction<Boolean>) () -> doPost(auditServerBaseUrl, auditEvent));
+ } catch (PrivilegedActionException e) {
+ Throwable cause = e.getCause() != null ? e.getCause() : e;
+
+ LOG.error("Failed to post audit event to ingestor at {}: {}", auditServerBaseUrl, cause.getMessage(), cause);
+
+ return false;
+ } catch (RuntimeException e) {
+ LOG.error("Failed to post audit event to ingestor at {}: {}", auditServerBaseUrl, e.getMessage(), e);
+
+ return false;
+ }
+ }
+
+ private static void initKerberos(Configuration config) throws IOException {
+ String principal = config.get(JAAS_PRINCIPAL_PROP);
+ String keytab = config.get(JAAS_KEYTAB_PROP);
+
+ Configuration hadoopConf = new Configuration(false);
+
+ hadoopConf.set("hadoop.security.authentication", "kerberos");
+ UserGroupInformation.setConfiguration(hadoopConf);
+ System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
+ UserGroupInformation.loginUserFromKeytab(principal, keytab);
+
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+
+ LOG.info("Kerberos initialized for audit ingestor client. Principal: {}", loginUser != null ? loginUser.getUserName() : principal);
+ }
+
+ private static boolean isKerberosConfigured(Configuration config) {
+ String principal = config.get(JAAS_PRINCIPAL_PROP);
+ String keytab = config.get(JAAS_KEYTAB_PROP);
+
+ return StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(keytab) && !"%EMPTY%".equalsIgnoreCase(principal);
+ }
+
+ private static boolean doPost(String auditServerBaseUrl, AuthzAuditEvent auditEvent) throws Exception {
+ if (kerberosInitialized && UserGroupInformation.isSecurityEnabled()) {
+ UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
+
+ if (loginUser != null && loginUser.hasKerberosCredentials()) {
+ return loginUser.doAs((PrivilegedExceptionAction<Boolean>) () -> doPostConnection(auditServerBaseUrl, auditEvent));
+ }
+ }
+
+ return doPostConnection(auditServerBaseUrl, auditEvent);
+ }
+
+ private static boolean doPostConnection(String auditServerBaseUrl, AuthzAuditEvent auditEvent) throws Exception {
+ String baseUrl = auditServerBaseUrl.endsWith("/") ? auditServerBaseUrl.substring(0, auditServerBaseUrl.length() - 1) : auditServerBaseUrl;
+ String service = auditEvent.getRepositoryName();
+ String appId = auditEvent.getAgentId();
+ StringBuilder urlBuilder = new StringBuilder(baseUrl)
+ .append(REST_PATH_POST)
+ .append('?')
+ .append(QUERY_PARAM_SERVICE)
+ .append('=')
+ .append(URLEncoder.encode(service, StandardCharsets.UTF_8.name()));
+
+ if (StringUtils.isNotBlank(appId)) {
+ urlBuilder.append('&')
+ .append(QUERY_PARAM_APP_ID)
+ .append('=')
+ .append(URLEncoder.encode(appId, StandardCharsets.UTF_8.name()));
+ }
+
+ List<AuthzAuditEvent> payload = Collections.singletonList(auditEvent);
+ byte[] body = OBJECT_MAPPER.writeValueAsBytes(payload);
+
+ URL requestUrl = new URL(urlBuilder.toString());
+ HttpURLConnection connection = (HttpURLConnection) requestUrl.openConnection();
+
+ connection.setConnectTimeout(30_000);
+ connection.setReadTimeout(30_000);
+ connection.setRequestMethod("POST");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestProperty("Accept", "application/json");
+
+ if (kerberosInitialized) {
+ String negotiateToken = createSpnegoToken(requestUrl.getHost());
+
+ if (StringUtils.isNotBlank(negotiateToken)) {
+ connection.setRequestProperty(AUTHZ_HEADER, NEGOTIATE_PREFIX + negotiateToken);
+ }
+ }
+
+ try (OutputStream outputStream = connection.getOutputStream()) {
+ outputStream.write(body);
+ }
+
+ int status = connection.getResponseCode();
+
+ if (status == HttpStatus.SC_OK) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Audit event posted to ingestor for service={} user={}", service, auditEvent.getUser());
+ }
+
+ return true;
+ }
+
+ LOG.error("Failed to post audit event to ingestor. HTTP status: {}", status);
+
+ return false;
+ }
+
+ private static String createSpnegoToken(String host) throws Exception {
+ GSSManager manager = GSSManager.getInstance();
+ GSSName server = manager.createName("HTTP@" + host, GSSName.NT_HOSTBASED_SERVICE);
+ Oid mech = resolveSpnegoMechanism(manager);
+
+ GSSContext context = manager.createContext(server, mech, null, GSSContext.DEFAULT_LIFETIME);
+
+ context.requestMutualAuth(false);
+ context.requestCredDeleg(false);
+
+ byte[] token = context.initSecContext(new byte[0], 0, 0);
+
+ if (token == null || token.length == 0) {
+ return null;
+ }
+
+ return Base64.getEncoder().encodeToString(token);
+ }
+
+ private static Oid resolveSpnegoMechanism(GSSManager manager) throws Exception {
+ Oid spnego = new Oid("1.3.6.1.5.5.14");
+
+ for (Oid mech : manager.getMechs()) {
+ if (spnego.equals(mech)) {
+ return spnego;
+ }
+ }
+
+ return new Oid("1.2.840.113554.1.2.2");
+ }
+}
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java
index 7784a5f92..6b27538d3 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuditHandler.java
@@ -3,8 +3,8 @@
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
@@ -20,27 +20,39 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.ranger.audit.model.AuthzAuditEvent;
-import org.apache.ranger.plugin.audit.RangerMultiResourceAuditHandler;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.List;
-public class RangerElasticsearchAuditHandler extends RangerMultiResourceAuditHandler {
+public class RangerElasticsearchAuditHandler extends RangerDefaultAuditHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuditHandler.class);
+
private static final String PROP_ES_PLUGIN_AUDIT_EXCLUDED_USERS = "ranger.elasticsearch.plugin.audit.excluded.users";
private static final String PROP_ES_PLUGIN_AUDIT_INDEX = "xasecure.audit.destination.elasticsearch.index";
+ private static final String PROP_AUDITSERVER_URL = "xasecure.audit.destination.auditserver.url";
private final String indexName;
private final List<String> excludeUsers;
+ private final String auditServerUrl;
public RangerElasticsearchAuditHandler(Configuration config) {
+ super(config);
+
String esUser = "elasticsearch";
String excludeUserList = config.get(PROP_ES_PLUGIN_AUDIT_EXCLUDED_USERS, esUser);
- excludeUsers = Arrays.asList(excludeUserList.split(","));
- indexName = config.get(PROP_ES_PLUGIN_AUDIT_INDEX, "ranger_audits");
+ excludeUsers = Arrays.asList(excludeUserList.split(","));
+ indexName = config.get(PROP_ES_PLUGIN_AUDIT_INDEX, "ranger_audits");
+ auditServerUrl = config.get(PROP_AUDITSERVER_URL);
+
+ ElasticsearchAuditIngestorClient.init(config);
}
@Override
@@ -53,10 +65,16 @@ public void processResult(RangerAccessResult result) {
AuthzAuditEvent auditEvent = super.getAuthzEvents(result);
- super.logAuthzAudit(auditEvent);
+ if (auditEvent != null && !ElasticsearchAuditIngestorClient.post(auditServerUrl, auditEvent)) {
+ MiscUtil.logErrorMessageByInterval(LOG, "fail to log audit event " + auditEvent);
+ }
}
private boolean isAuditingNeeded(final RangerAccessResult result) {
+ if (result == null) {
+ return false;
+ }
+
boolean ret = true;
boolean isAllowed = result.getIsAllowed();
RangerAccessRequest request = result.getAccessRequest();
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
index f05ab9ff0..f0dd4e041 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
@@ -21,6 +21,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
@@ -30,6 +31,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -39,9 +41,16 @@ public class RangerElasticsearchAuthorizer implements RangerElasticsearchAccessC
private static volatile RangerElasticsearchInnerPlugin elasticsearchPlugin;
+ private final String configDir;
+
public RangerElasticsearchAuthorizer() {
+ this(null);
+ }
+
+ public RangerElasticsearchAuthorizer(String configDir) {
LOG.debug("==> RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()");
+ this.configDir = configDir;
this.init();
LOG.debug("<== RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()");
@@ -57,7 +66,7 @@ public void init() {
plugin = elasticsearchPlugin;
if (plugin == null) {
- plugin = new RangerElasticsearchInnerPlugin();
+ plugin = new RangerElasticsearchInnerPlugin(configDir);
plugin.init();
@@ -95,8 +104,23 @@ public boolean checkPermission(String user, List<String> groups, String index, S
}
static class RangerElasticsearchInnerPlugin extends RangerBasePlugin {
- public RangerElasticsearchInnerPlugin() {
- super("elasticsearch", "elasticsearch");
+ public RangerElasticsearchInnerPlugin(String configDir) {
+ super(createPluginConfig(configDir));
+ }
+
+ private static RangerPluginConfig createPluginConfig(String configDir) {
+ List<File> additionalConfigFiles = null;
+
+ if (configDir != null) {
+ File dir = new File(configDir);
+
+ additionalConfigFiles = new ArrayList<>();
+ additionalConfigFiles.add(new File(dir, "ranger-elasticsearch-audit.xml"));
+ additionalConfigFiles.add(new File(dir, "ranger-elasticsearch-security.xml"));
+ additionalConfigFiles.add(new File(dir, "ranger-policymgr-ssl.xml"));
+ }
+
+ return new RangerPluginConfig("elasticsearch", null, "elasticsearch", null, null, additionalConfigFiles, null);
}
@Override
diff --git a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
index f3d07c0b9..88280597e 100644
--- a/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
+++ b/plugin-elasticsearch/src/main/java/org/apache/ranger/services/elasticsearch/RangerServiceElasticsearch.java
@@ -40,7 +40,7 @@ public class RangerServiceElasticsearch extends RangerBaseService {
public static final String ACCESS_TYPE_READ = "read";
- private RangerServiceElasticsearch() {
+ public RangerServiceElasticsearch() {
super();
}
diff --git a/pom.xml b/pom.xml
index 61140d180..02198d72d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
<distMgmtStagingUrl>https://repository.apache.org/service/local/staging/deploy/maven2</distMgmtStagingUrl>
<dnsjava.version>3.6.2</dnsjava.version>
<eclipse.jpa.version>2.7.12</eclipse.jpa.version>
+ <elasticsearch.jackson.version>2.14.2</elasticsearch.jackson.version>
<elasticsearch.version>7.17.29</elasticsearch.version>
<enunciate.version>2.15.0</enunciate.version>
<fasterxml.jackson.databind.version>2.18.9</fasterxml.jackson.databind.version>
diff --git a/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties b/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties
index 95f9604aa..1af4c62f6 100644
--- a/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties
+++ b/ranger-elasticsearch-plugin-shim/conf/plugin-descriptor.properties
@@ -56,10 +56,7 @@ elasticsearch.version=${elasticsearch.version}
### optional elements for plugins:
#
# 'extended.plugins': other plugins this plugin extends through SPI
-#extended.plugins=${extendedPlugins}
+extended.plugins=x-pack-security
#
# 'has.native.controller': whether or not the plugin has a native controller
has.native.controller=false
-#
-# 'requires.keystore': whether or not the plugin needs the elasticsearch keystore be created
-requires.keystore=false
diff --git a/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy b/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy
index 4f96d87f5..b4de434f2 100644
--- a/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy
+++ b/ranger-elasticsearch-plugin-shim/conf/plugin-security.policy
@@ -9,31 +9,44 @@
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
+// Bare grant applies to all jars in this plugin directory (shim, impl, Hadoop, Ranger).
+// ES 7.17 rejects AllPermission here; permissions must match PolicyUtil.ALLOWED_PLUGIN_PERMISSIONS.
grant {
- permission java.lang.RuntimePermission "createClassLoader";
+ // Hadoop Configuration / UGI static init (same as repository-hdfs plugin)
permission java.lang.RuntimePermission "getClassLoader";
- permission java.lang.RuntimePermission "setContextClassLoader";
- permission java.lang.RuntimePermission "shutdownHooks";
permission java.lang.RuntimePermission "accessDeclaredMembers";
- permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.lang.RuntimePermission "setContextClassLoader";
+ permission java.util.PropertyPermission "*", "read,write";
- permission javax.security.auth.AuthPermission "getLoginConfiguration";
- permission javax.security.auth.AuthPermission "setLoginConfiguration";
+ // JAAS / Kerberos (Hadoop auth stack)
+ permission javax.security.auth.AuthPermission "getSubject";
+ permission javax.security.auth.AuthPermission "doAs";
+ permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
+ permission java.lang.RuntimePermission "accessClassInPackage.sun.security.krb5";
+ permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
+ permission javax.security.auth.AuthPermission "modifyPrincipals";
+ permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
+ permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
+ permission java.lang.RuntimePermission "loadLibrary.jaas";
+ permission java.lang.RuntimePermission "loadLibrary.jaas_unix";
+ permission java.lang.RuntimePermission "loadLibrary.jaas_nt";
+ permission javax.security.auth.AuthPermission "modifyPublicCredentials";
+ permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
+ permission java.security.SecurityPermission "insertProvider";
+ permission javax.security.auth.kerberos.ServicePermission "*", "initiate";
- permission java.net.NetPermission "getProxySelector";
- // adapt to connect different IP and Port
+ // Ranger plugin: policy download from Admin, audit ingestor HTTP POST
+ permission java.net.SocketPermission "ranger-audit-ingestor.rangernw:7081", "connect,resolve";
+ permission java.net.SocketPermission "ranger.rangernw:6080", "connect,resolve";
permission java.net.SocketPermission "*", "connect,resolve";
-
- permission java.util.PropertyPermission "*", "read,write";
- // adapt to different directories configured by user
- permission java.io.FilePermission "<<ALL FILES>>", "read,write";
+ permission java.net.SocketPermission "localhost:0", "listen,resolve";
+ permission java.io.FilePermission "<<ALL FILES>>", "read";
};
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
deleted file mode 100644
index c5290013c..000000000
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizer.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ranger.authorization.elasticsearch.authorizer;
-
-import org.apache.ranger.plugin.classloader.RangerPluginClassLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public class RangerElasticsearchAuthorizer {
- private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizer.class);
-
- private static final String RANGER_PLUGIN_TYPE = "elasticsearch";
- private static final String RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME = "org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer";
-
- private RangerPluginClassLoader rangerPluginClassLoader;
- private ClassLoader esClassLoader;
- private RangerElasticsearchAccessControl rangerElasticsearchAccessControl;
-
- public RangerElasticsearchAuthorizer() {
- LOG.debug("==> RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()");
-
- this.init();
-
- LOG.debug("<== RangerElasticsearchAuthorizer.RangerElasticsearchAuthorizer()");
- }
-
- public void init() {
- LOG.debug("==> RangerElasticsearchAuthorizer.init()");
-
- try {
- // In elasticsearch this.getClass().getClassLoader() is FactoryURLClassLoader,
- // but Thread.currentThread().getContextClassLoader() is AppClassLoader.
- esClassLoader = Thread.currentThread().getContextClassLoader();
-
- Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
-
- rangerPluginClassLoader = RangerPluginClassLoader.getInstance(RANGER_PLUGIN_TYPE, this.getClass());
-
- Thread.currentThread().setContextClassLoader(esClassLoader);
-
- @SuppressWarnings("unchecked")
- Class<RangerElasticsearchAccessControl> cls = (Class<RangerElasticsearchAccessControl>) Class.forName(RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME, true, rangerPluginClassLoader);
-
- activatePluginClassLoader();
-
- rangerElasticsearchAccessControl = cls.newInstance();
- } catch (Exception e) {
- LOG.error("Error Enabling RangerElasticsearchAuthorizer", e);
- } finally {
- deactivatePluginClassLoader();
- }
-
- LOG.debug("<== RangerElasticsearchAuthorizer.init()");
- }
-
- public boolean checkPermission(String user, List<String> groups, String index, String action, String clientIPAddress) {
- boolean ret;
-
- LOG.debug("==> RangerElasticsearchAuthorizer.checkPermission()");
-
- try {
- activatePluginClassLoader();
-
- ret = rangerElasticsearchAccessControl.checkPermission(user, groups, index, action, clientIPAddress);
- } finally {
- deactivatePluginClassLoader();
- }
-
- LOG.debug("<== RangerElasticsearchAuthorizer.checkPermission()");
-
- return ret;
- }
-
- private void activatePluginClassLoader() {
- if (rangerPluginClassLoader != null) {
- Thread.currentThread().setContextClassLoader(rangerPluginClassLoader);
- }
- }
-
- private void deactivatePluginClassLoader() {
- if (esClassLoader != null) {
- Thread.currentThread().setContextClassLoader(esClassLoader);
- }
- }
-}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java
new file mode 100644
index 000000000..c7e21041b
--- /dev/null
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/authorizer/RangerElasticsearchAuthorizerDelegate.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.authorization.elasticsearch.authorizer;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+/**
+ * Shim-side delegate that loads the real {@link RangerElasticsearchAuthorizer} implementation
+ * from the Elasticsearch plugin classloader. ES 7.17+ does not grant plugins
+ * {@code createClassLoader}, so we avoid {@code RangerPluginClassLoader} here and rely on
+ * impl jars being on the plugin classpath.
+ */
+public class RangerElasticsearchAuthorizerDelegate {
+ private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchAuthorizerDelegate.class);
+
+ private static final String RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME =
+ "org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer";
+
+ private RangerElasticsearchAccessControl rangerElasticsearchAccessControl;
+
+ private final String configDir;
+
+ public RangerElasticsearchAuthorizerDelegate(String configDir) {
+ LOG.debug("==> RangerElasticsearchAuthorizerDelegate()");
+
+ this.configDir = configDir;
+ this.init();
+
+ LOG.debug("<== RangerElasticsearchAuthorizerDelegate()");
+ }
+
+ public void init() {
+ LOG.debug("==> RangerElasticsearchAuthorizerDelegate.init()");
+
+ try {
+ @SuppressWarnings("unchecked")
+ Class<RangerElasticsearchAccessControl> cls = (Class<RangerElasticsearchAccessControl>) Class.forName(
+ RANGER_ELASTICSEARCH_AUTHORIZER_IMPL_CLASSNAME);
+
+ rangerElasticsearchAccessControl = java.security.AccessController.doPrivileged(
+ (PrivilegedExceptionAction<RangerElasticsearchAccessControl>) () -> cls.getDeclaredConstructor(String.class)
+ .newInstance(configDir));
+ } catch (Exception e) {
+ LOG.error("Error Enabling RangerElasticsearchAuthorizer", e);
+ }
+
+ LOG.debug("<== RangerElasticsearchAuthorizerDelegate.init()");
+ }
+
+ public boolean checkPermission(String user, List<String> groups, String index, String action, String clientIPAddress) {
+ LOG.debug("==> RangerElasticsearchAuthorizerDelegate.checkPermission()");
+
+ if (rangerElasticsearchAccessControl == null) {
+ LOG.warn("RangerElasticsearchAuthorizer is not initialized; denying access.");
+
+ return false;
+ }
+
+ boolean ret;
+
+ try {
+ ret = java.security.AccessController.doPrivileged(
+ (PrivilegedExceptionAction<Boolean>) () -> rangerElasticsearchAccessControl.checkPermission(
+ user, groups, index, action, clientIPAddress));
+ } catch (Exception e) {
+ LOG.error("Error checking Ranger permission for user[{}] action[{}] index[{}]", user, action, index, e);
+
+ ret = false;
+ }
+
+ LOG.debug("<== RangerElasticsearchAuthorizerDelegate.checkPermission()");
+
+ return ret;
+ }
+}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
index e3b740f00..5d69fd5dd 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/RangerElasticsearchPlugin.java
@@ -17,8 +17,8 @@
package org.apache.ranger.authorization.elasticsearch.plugin;
+import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizerDelegate;
import org.apache.ranger.authorization.elasticsearch.plugin.action.filter.RangerSecurityActionFilter;
-import org.apache.ranger.authorization.elasticsearch.plugin.rest.filter.RangerSecurityRestFilter;
import org.elasticsearch.action.support.ActionFilter;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -31,7 +31,6 @@
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoriesService;
-import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
@@ -39,17 +38,18 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.net.URLClassLoader;
+import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Supplier;
-import java.util.function.UnaryOperator;
+/**
+ * Ranger authorization integrates with X-Pack Security: authentication and REST handling
+ * are delegated to x-pack-security; Ranger enforces policies via {@link RangerSecurityActionFilter}
+ * using the verified user from {@link org.elasticsearch.xpack.core.security.SecurityContext}.
+ */
public class RangerElasticsearchPlugin extends Plugin implements ActionPlugin {
private static final Logger LOG = LoggerFactory.getLogger(RangerElasticsearchPlugin.class);
@@ -70,57 +70,48 @@ public List<ActionFilter> getActionFilters() {
return Collections.singletonList(rangerSecurityActionFilter);
}
- @Override
- public UnaryOperator<RestHandler> getRestHandlerWrapper(ThreadContext threadContext) {
- return handler -> new RangerSecurityRestFilter(settings, threadContext, handler);
- }
-
@Override
public Collection<Object> createComponents(final Client client, final ClusterService clusterService, final ThreadPool threadPool, final ResourceWatcherService resourceWatcherService,
final ScriptService scriptService, final NamedXContentRegistry xContentRegistry, final Environment environment, final NodeEnvironment nodeEnvironment,
final NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<RepositoriesService> repositoriesServiceSupplier) {
- addPluginConfig2Classpath(environment);
+ Path configPath = registerPluginConfigDir(environment);
- rangerSecurityActionFilter = new RangerSecurityActionFilter(settings, threadPool.getThreadContext());
+ RangerElasticsearchAuthorizerDelegate authorizer = initAuthorizer(threadPool.getThreadContext(), configPath);
+
+ rangerSecurityActionFilter = new RangerSecurityActionFilter(settings, threadPool.getThreadContext(), authorizer);
return Collections.singletonList(rangerSecurityActionFilter);
}
/**
- * Add ranger elasticsearch plugin config directory to classpath,
- * then the plugin can load its configuration files from classpath.
+ * Initialize Ranger in Elasticsearch system context so Hadoop/Ranger plugin setup is not
+ * blocked by the plugin security manager during node startup.
*/
- private void addPluginConfig2Classpath(Environment environment) {
- Path configPath = environment.configFile().resolve(RANGER_ELASTICSEARCH_PLUGIN_CONF_NAME);
+ private RangerElasticsearchAuthorizerDelegate initAuthorizer(ThreadContext threadContext, Path configPath) {
+ String configDir = configPath != null ? configPath.toAbsolutePath().toString() : null;
- if (configPath == null) {
- LOG.error("Failed to add ranger elasticsearch plugin config directory [ranger-elasticsearch-plugin] to classpath.");
+ try (ThreadContext.StoredContext ignored = threadContext.stashContext()) {
+ threadContext.markAsSystemContext();
- return;
+ return new RangerElasticsearchAuthorizerDelegate(configDir);
}
+ }
- File configFile = configPath.toFile();
-
- try {
- if (configFile.exists()) {
- ClassLoader classLoader = this.getClass().getClassLoader();
+ /**
+ * Resolve the on-disk Ranger config directory for the authorizer implementation.
+ * ES 7.17 on Java 17 cannot extend the plugin classloader via reflection.
+ */
+ private Path registerPluginConfigDir(Environment environment) {
+ Path configPath = environment.configFile().resolve(RANGER_ELASTICSEARCH_PLUGIN_CONF_NAME);
- // This classLoader is FactoryURLClassLoader in elasticsearch
- if (classLoader instanceof URLClassLoader) {
- URLClassLoader urlClassLoader = (URLClassLoader) classLoader;
- Class<? extends URLClassLoader> urlClass = urlClassLoader.getClass();
- Method method = urlClass.getSuperclass().getDeclaredMethod("addURL", URL.class);
+ if (!Files.isDirectory(configPath)) {
+ LOG.error("Ranger elasticsearch plugin config directory [{}] does not exist.", configPath);
- method.setAccessible(true);
- method.invoke(urlClassLoader, configFile.toURI().toURL());
+ return null;
+ }
- LOG.info("Success to add ranger elasticsearch plugin config directory [{}] to classpath.", configFile.getCanonicalPath());
- }
- }
- } catch (Exception e) {
- LOG.error("Failed to add ranger elasticsearch plugin config directory [ranger-elasticsearch-plugin] to classpath.", e);
+ LOG.info("Using Ranger elasticsearch plugin config directory [{}].", configPath);
- throw new RuntimeException(e);
- }
+ return configPath;
}
}
diff --git a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
index 1d70e3b0a..528db15e3 100644
--- a/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
+++ b/ranger-elasticsearch-plugin-shim/src/main/java/org/apache/ranger/authorization/elasticsearch/plugin/action/filter/RangerSecurityActionFilter.java
@@ -18,7 +18,7 @@
package org.apache.ranger.authorization.elasticsearch.plugin.action.filter;
import org.apache.commons.lang3.StringUtils;
-import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizer;
+import org.apache.ranger.authorization.elasticsearch.authorizer.RangerElasticsearchAuthorizerDelegate;
import org.apache.ranger.authorization.elasticsearch.plugin.authc.ElasticsearchAuthenticatedUserResolver;
import org.apache.ranger.authorization.elasticsearch.plugin.authc.user.UsernamePasswordToken;
import org.apache.ranger.authorization.elasticsearch.plugin.utils.RequestUtils;
@@ -43,18 +43,23 @@ public class RangerSecurityActionFilter extends AbstractLifecycleComponent imple
private final Settings settings;
private final ThreadContext threadContext;
- private final RangerElasticsearchAuthorizer rangerElasticsearchAuthorizer = new RangerElasticsearchAuthorizer();
+ private final RangerElasticsearchAuthorizerDelegate rangerElasticsearchAuthorizer;
- public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext) {
+ public RangerSecurityActionFilter(Settings settings, ThreadContext threadContext, RangerElasticsearchAuthorizerDelegate rangerElasticsearchAuthorizer) {
super();
- this.settings = settings;
- this.threadContext = threadContext;
+ this.settings = settings;
+ this.threadContext = threadContext;
+ this.rangerElasticsearchAuthorizer = rangerElasticsearchAuthorizer;
}
+ /**
+ * Run after x-pack {@code SecurityActionFilter} (order {@code Integer.MIN_VALUE}) so
+ * {@link org.elasticsearch.xpack.core.security.SecurityContext} holds the verified user.
+ */
@Override
public int order() {
- return 0;
+ return Integer.MAX_VALUE;
}
@Override
@@ -73,6 +78,11 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
}
}
+ if (shouldBypassRangerCheck(user, action)) {
+ chain.proceed(task, action, request, listener);
+ return;
+ }
+
if (StringUtils.isNotEmpty(user)) {
List<String> indexs = RequestUtils.getIndexFromRequest(request);
String clientIPAddress = threadContext.getTransient(RequestUtils.CLIENT_IP_ADDRESS);
@@ -86,8 +96,6 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
throw new ElasticsearchStatusException(errorMsg, RestStatus.FORBIDDEN, user, action, index);
}
}
- } else if (threadContext.isSystemContext()) {
- LOG.debug("System context request, skipping Ranger permission check for action[{}].", action);
} else {
throw new ElasticsearchStatusException("Error: Request requires authenticated user.", RestStatus.UNAUTHORIZED);
}
@@ -95,6 +103,18 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
chain.proceed(task, action, request, listener);
}
+ private boolean shouldBypassRangerCheck(String user, String action) {
+ if (threadContext.isSystemContext()) {
+ return true;
+ }
+
+ if (StringUtils.isNotEmpty(user) && user.charAt(0) == '_') {
+ return true;
+ }
+
+ return StringUtils.isNotEmpty(action) && action.startsWith("internal:");
+ }
+
@Override
protected void doStart() {
}
diff --git a/ranger_in_docker b/ranger_in_docker
index 0db00f9aa..ecbdb89ab 100755
--- a/ranger_in_docker
+++ b/ranger_in_docker
@@ -42,7 +42,7 @@
# by defining comma separated optional services as following:
# ENABLED_RANGER_SERVICES="hadoop,hive,hbase,knox,kms"
# List of optional services:
-# tagsync,hadoop,hbase,kafka,hive,knox,kms
+# tagsync,hadoop,hbase,kafka,hive,knox,kms,elasticsearch
#
if [ -z "${RANGER_HOME}" ]
then