You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2023/06/28 23:56:25 UTC
[ranger] branch master updated: RANGER-4303: plugin memory sizing tool
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new e6b0f880a RANGER-4303: plugin memory sizing tool
e6b0f880a is described below
commit e6b0f880a30be0767deb78c4136c4c8825a1d732
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Sun Jun 25 22:20:46 2023 -0700
RANGER-4303: plugin memory sizing tool
---
distro/src/main/assembly/ranger-tools.xml | 6 +
ranger-tools/conf/logback-mem-sizing.xml | 45 ++
ranger-tools/scripts/ranger-mem-sizing.sh | 46 ++
.../apache/ranger/sizing/PerfMemTimeTracker.java | 140 ++++++
.../org/apache/ranger/sizing/RangerMemSizing.java | 470 +++++++++++++++++++++
ranger-tools/testdata/test_roles_hive.json | 235 +++++++++++
ranger-tools/testdata/test_userstore_hive.json | 69 +++
7 files changed, 1011 insertions(+)
diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml
index b8713d806..794bddb51 100644
--- a/distro/src/main/assembly/ranger-tools.xml
+++ b/distro/src/main/assembly/ranger-tools.xml
@@ -134,6 +134,12 @@
<destName>ranger-plugin-perftester.sh</destName>
<fileMode>755</fileMode>
</file>
+ <file>
+ <source>${project.parent.basedir}/ranger-tools/scripts/ranger-mem-sizing.sh</source>
+ <outputDirectory></outputDirectory>
+ <destName>ranger-mem-sizing.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
<file>
<source>${project.parent.basedir}/ranger-tools/scripts/README.txt</source>
<outputDirectory></outputDirectory>
diff --git a/ranger-tools/conf/logback-mem-sizing.xml b/ranger-tools/conf/logback-mem-sizing.xml
new file mode 100644
index 000000000..c3b601451
--- /dev/null
+++ b/ranger-tools/conf/logback-mem-sizing.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>./ranger-mem-sizing-${hostname}-${user}.log</file>
+ <append>true</append>
+ <encoder>
+ <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>./ranger-mem-sizing-${hostname}-${user}.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>15</maxHistory>
+ <cleanHistoryOnStart>true</cleanHistoryOnStart>
+ </rollingPolicy>
+ </appender>
+
+ <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+ <Target>System.out</Target>
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p - %m%n</pattern>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+ <root level="INFO">
+ <appender-ref ref="file"/>
+ </root>
+</configuration>
diff --git a/ranger-tools/scripts/ranger-mem-sizing.sh b/ranger-tools/scripts/ranger-mem-sizing.sh
new file mode 100755
index 000000000..c147c96f3
--- /dev/null
+++ b/ranger-tools/scripts/ranger-mem-sizing.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Review and update following variables as needed
+#
+RANGER_ADMIN_HOME=${RANGER_ADMIN_HOME:-/opt/ranger/admin}
+MEMORY=1g
+
+#
+# Usage:
+# ranger-mem-sizing.sh -p policies.json -t tags.json -u userstore.json -r roles.json
+#
+
+#
+#
+#
+cdir=$(cd "$(dirname "$0")"; pwd)
+cp="${cdir}/dist/*:${cdir}/lib/commons-cli-1.3.1.jar:${RANGER_ADMIN_HOME}/ews/lib/*:${RANGER_ADMIN_HOME}/ews/webapp/WEB-INF/lib/*"
+
+if [ "${JAVA_HOME}" != "" ]
+then
+ export JAVA_HOME
+ PATH="${JAVA_HOME}/bin:${PATH}"
+ export PATH
+fi
+
+JAVA_CMD="java -Xms${MEMORY} -Xmx${MEMORY} -Xloggc:./gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=. -Dlogback.configurationFile=file:${cdir}/conf/logback-mem-sizing.xml -cp ${cp} org.apache.ranger.sizing.RangerMemSizing"
+
+cd ${cdir}
+
+echo "JAVA command = $JAVA_CMD " "$@"
+$JAVA_CMD "$@"
diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java
new file mode 100644
index 000000000..917c0da9c
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.sizing;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PerfMemTimeTracker {
+ public static final String STR_INTENT = " ";
+ public static final String STR_FIELD_SEP = "|";
+
+ private final String tag;
+ private final long startTime;
+ private long endTime;
+ private long gcTime;
+ private final long startMemory;
+ private long endMemory;
+ private List<PerfMemTimeTracker> children = null;
+
+ public PerfMemTimeTracker(String tag) {
+ this.tag = tag;
+ this.startTime = System.currentTimeMillis();
+
+ Runtime rt = Runtime.getRuntime();
+
+ rt.gc();
+
+ this.gcTime = System.currentTimeMillis() - startTime;
+ this.startMemory = rt.totalMemory() - rt.freeMemory();
+ }
+
+ public void stop() {
+ long gcStartTime = System.currentTimeMillis();
+
+ Runtime rt = Runtime.getRuntime();
+
+ rt.gc();
+
+ this.endTime = System.currentTimeMillis();
+ this.gcTime += endTime - gcStartTime;
+ this.endMemory = rt.totalMemory() - rt.freeMemory();
+ }
+
+ public void addChild(PerfMemTimeTracker tracker) {
+ if (children == null) {
+ children = new ArrayList<>();
+ }
+
+ children.add(tracker);
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public long getGcTime() {
+ return gcTime;
+ }
+
+ public long getStartMemory() {
+ return startMemory;
+ }
+
+ public long getEndMemory() {
+ return endMemory;
+ }
+
+ public List<PerfMemTimeTracker> getChildren() {
+ return children;
+ }
+
+ public long getTimeTaken() {
+ return endTime - startTime - getTotalGcTime();
+ }
+
+ public long getTotalGcTime() {
+ long ret = gcTime;
+
+ if (children != null) {
+ for (PerfMemTimeTracker child : children) {
+ ret += child.getTotalGcTime();
+ }
+ }
+
+ return ret;
+ }
+
+ public long getMemoryDelta() {
+ return endMemory - startMemory;
+ }
+
+ public void print(PrintStream out, boolean printHeader) {
+ if (printHeader) {
+ out.println("Task|Time (ms)|Memory (bytes)");
+ }
+
+ print("", out);
+ }
+
+ @Override
+ public String toString() {
+ return tag + ", Memory: (start: " + startMemory + ", end: " + endMemory + ", delta: " + getMemoryDelta() + ") bytes, TimeTaken: " + getTimeTaken() + "ms";
+ }
+
+ private void print(String intentString, PrintStream out) {
+ out.println(intentString + tag + STR_FIELD_SEP + getTimeTaken() + STR_FIELD_SEP + getMemoryDelta());
+
+ if (children != null) {
+ for (PerfMemTimeTracker child : children) {
+ child.print(intentString + STR_INTENT, out);
+ }
+ }
+ }
+}
diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
new file mode 100644
index 000000000..5b6f51339
--- /dev/null
+++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.sizing;
+
+import java.io.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Set;
+
+import org.apache.commons.cli.*;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
+import org.apache.ranger.plugin.service.RangerBasePlugin;
+import org.apache.ranger.plugin.util.RangerRoles;
+import org.apache.ranger.plugin.util.ServicePolicies;
+import org.apache.ranger.plugin.util.ServicePolicies.SecurityZoneInfo;
+import org.apache.ranger.plugin.util.ServiceTags;
+import org.apache.ranger.plugin.util.RangerUserStore;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+
+public class RangerMemSizing {
+ private static final String OPT_MODE_SPACE = "space";
+ private static final String OPT_MODEL_RETRIEVAL = "retrieval";
+
+ private final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S");
+
+ private final Gson gson;
+ private final String policyFile;
+ private final String tagFile;
+ private final String rolesFile;
+ private final String userStoreFile;
+ private final boolean deDup;
+ private final boolean deDupStrings;
+ private final String optimizationMode;
+ private final PrintStream out;
+
+ public RangerMemSizing(CommandLine cmdLine) {
+ this.out = System.out;
+ this.gson = createGson();
+ this.policyFile = cmdLine.getOptionValue('p');
+ this.tagFile = cmdLine.getOptionValue('t');
+ this.rolesFile = cmdLine.getOptionValue('r');
+ this.userStoreFile = cmdLine.getOptionValue('u');
+ this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d", "true"));
+ this.deDupStrings = this.deDup;
+ this.optimizationMode = StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ? OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL;
+ }
+
+ public void run() {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("RangerMemSizing");
+
+ ServicePolicies policies = loadPolicies(policyFile, tracker);
+ ServiceTags tags = loadTags(tagFile, tracker);
+ RangerRoles roles = loadRoles(rolesFile, tracker);
+ RangerUserStore userStore = loadUserStore(userStoreFile, tracker);
+ RangerBasePlugin plugin = createRangerPlugin(policies, tags, roles, userStore, tracker);
+
+ tracker.stop();
+
+ out.println();
+ out.println("Parameters:");
+ if (policies != null) {
+ out.println(" Policies: file=" + policyFile + ", size=" + new File(policyFile).length() + ", " + toSummaryStr(policies));
+ }
+
+ if (tags != null) {
+ out.println(" Tags: file=" + tagFile + ", size=" + new File(tagFile).length() + ", " + toSummaryStr(tags));
+ }
+
+ if (roles != null) {
+ out.println(" Roles: file=" + rolesFile + ", size=" + new File(rolesFile).length() + ", " + toSummaryStr(roles));
+ }
+
+ if (userStore != null) {
+ out.println(" UserStore: file=" + userStoreFile + ", size=" + new File(userStoreFile).length() + ", " + toSummaryStr(userStore));
+ }
+
+ out.println(" DeDup: " + deDup);
+ out.println(" OptMode: " + optimizationMode);
+ out.println();
+
+ out.println("Results:");
+ out.println("*****************************");
+ tracker.print(out, true);
+ out.println("*****************************");
+ }
+
+ public static void main(String[] args) {
+ CommandLine cmdLine = parseArgs(args);
+
+ if (cmdLine != null) {
+ RangerMemSizing memSizing = new RangerMemSizing(cmdLine);
+
+ memSizing.run();
+ }
+ }
+
+ private ServicePolicies loadPolicies(String fileName, PerfMemTimeTracker parent) {
+ if (fileName == null) {
+ return null;
+ }
+
+ ServicePolicies ret = null;
+
+ try {
+ File file = new File(fileName);
+ PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load policies");
+
+ log("loading policies(file=" + fileName + ")");
+
+ {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read policies");
+
+ try (FileReader reader = new FileReader(file)) {
+ ret = gson.fromJson(reader, ServicePolicies.class);
+ }
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ if (deDupStrings) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings");
+
+ ret.dedupStrings();
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ loadTracker.stop();
+ parent.addChild(loadTracker);
+
+ log("loaded policies(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret));
+ } catch (FileNotFoundException excp) {
+ log(fileName + ": file does not exist!");
+ } catch (IOException excp) {
+ log(fileName, excp);
+ }
+
+ return ret;
+ }
+
+ private ServiceTags loadTags(String fileName, PerfMemTimeTracker parent) {
+ if (fileName == null) {
+ return null;
+ }
+
+ ServiceTags ret = null;
+
+ try {
+ File file = new File(fileName);
+ PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load tags");
+
+ log("loading tags(file=" + fileName + ")");
+
+ {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read tags");
+
+ try (FileReader reader = new FileReader(file)) {
+ ret = gson.fromJson(reader, ServiceTags.class);
+ }
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ if (deDup) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupTags");
+
+ int countOfDuplicateTags = ret.dedupTags();
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ log("DeDupTags(duplicateTags=" + countOfDuplicateTags + ")");
+ }
+
+ if (deDupStrings) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings");
+
+ ret.dedupStrings();
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ loadTracker.stop();
+ parent.addChild(loadTracker);
+
+ log("loaded tags(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret));
+ } catch (FileNotFoundException excp) {
+ log(fileName + ": file does not exist!");
+ } catch (IOException excp) {
+ log(fileName, excp);
+ }
+
+ return ret;
+ }
+
+ private RangerRoles loadRoles(String fileName, PerfMemTimeTracker parent) {
+ if (fileName == null) {
+ return null;
+ }
+
+ RangerRoles ret = null;
+
+ try {
+ File file = new File(fileName);
+ PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load roles");
+
+ log("loading roles(file=" + fileName + ")");
+
+ try (FileReader reader = new FileReader(file)) {
+ ret = gson.fromJson(reader, RangerRoles.class);
+ }
+
+ loadTracker.stop();
+ parent.addChild(loadTracker);
+
+ log("loaded roles(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret));
+ } catch (FileNotFoundException excp) {
+ log(fileName + ": file does not exist!");
+ } catch (IOException excp) {
+ log(fileName, excp);
+ }
+
+ return ret;
+ }
+
+ private RangerUserStore loadUserStore(String fileName, PerfMemTimeTracker parent) {
+ if (fileName == null) {
+ return null;
+ }
+
+ RangerUserStore ret = null;
+
+ try {
+ File file = new File(fileName);
+ PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load userStore");
+
+ log("loading userStore(file=" + fileName + ")");
+
+ {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read userStore");
+
+ try (FileReader reader = new FileReader(file)) {
+ ret = gson.fromJson(reader, RangerUserStore.class);
+ }
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ if (deDupStrings) {
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings");
+
+ ret.dedupStrings();
+
+ tracker.stop();
+ loadTracker.addChild(tracker);
+ }
+
+ loadTracker.stop();
+ parent.addChild(loadTracker);
+
+ log("loaded userStore(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret) + ")");
+ } catch (FileNotFoundException excp) {
+ log(fileName + ": file does not exist!");
+ } catch (IOException excp) {
+ log(fileName, excp);
+ }
+
+ return ret;
+ }
+
+ private RangerBasePlugin createRangerPlugin(ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore, PerfMemTimeTracker parent) {
+ RangerBasePlugin ret = null;
+
+ if (policies != null) {
+ String serviceType = policies.getServiceDef().getName();
+ String serviceName = policies.getServiceName();
+ RangerPluginConfig pluginConfig = new RangerPluginConfig(serviceType, serviceName, serviceType, null, null, getPolicyEngineOptions());
+
+ PerfMemTimeTracker tracker = new PerfMemTimeTracker("RangerBasePlugin initialization");
+
+ log("Initializing RangerBasePlugin...");
+
+ ret = new RangerBasePlugin(pluginConfig, policies, tags, roles, userStore);
+
+ tracker.stop();
+ parent.addChild(tracker);
+ log("Initialized RangerBasePlugin.");
+ }
+
+ return ret;
+ }
+
+ private static CommandLine parseArgs(String[] args) {
+ Option help = new Option("h", "help", false, "show help");
+ Option deDup = new Option("d", "deDup", true, "deDup string/tags");
+ Option policies = new Option("p", "policies", true, "policies file");
+ Option tags = new Option("t", "tags", true, "tags file");
+ Option roles = new Option("r", "roles", true, "roles file");
+ Option userStore = new Option("u", "userStore", true, "userStore file");
+ Option optimizeMode = new Option("o", "optMode", true, "optimization mode: space|retrieval");
+
+ Options options = new Options();
+
+ options.addOption(help);
+ options.addOption(policies);
+ options.addOption(tags);
+ options.addOption(roles);
+ options.addOption(userStore);
+ options.addOption(deDup);
+ options.addOption(optimizeMode);
+
+ try {
+ CommandLine cmdLine = new DefaultParser().parse(options, args);
+
+ if (! cmdLine.hasOption("h")) {
+ return cmdLine;
+ }
+
+ new HelpFormatter().printHelp("RangerMemSizing", options);
+ } catch (ParseException excp) {
+ System.out.println("Failed to parse arguments");
+ excp.printStackTrace(System.out);
+ }
+
+ return null;
+ }
+
+ private Gson createGson() {
+ Gson gson = null;
+
+ try {
+ gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
+ } catch(Throwable excp) {
+ log("failed to create GsonBuilder object", excp);
+ }
+
+ return gson;
+ }
+
+ private void log(String msg) {
+ out.println(DATE_FORMAT.format(new Date()) + ": " +msg);
+ }
+
+ private void log(String msg, Throwable excp) {
+ out.println(DATE_FORMAT.format(new Date()) + ": " +msg);
+
+ excp.printStackTrace(out);
+ }
+
+ private RangerPolicyEngineOptions getPolicyEngineOptions() {
+ RangerPolicyEngineOptions ret = new RangerPolicyEngineOptions();
+
+ ret.disablePolicyRefresher = true;
+ ret.disableTagRetriever = true;
+ ret.disableUserStoreRetriever = true;
+ ret.optimizeTrieForSpace = optimizationMode.equals(OPT_MODE_SPACE);
+ ret.optimizeTrieForRetrieval = !ret.optimizeTrieForSpace;
+ ret.optimizeTagTrieForSpace = ret.optimizeTrieForSpace;
+ ret.optimizeTagTrieForRetrieval = ret.optimizeTrieForRetrieval;
+
+ return ret;
+ }
+
+ private static String toSummaryStr(ServicePolicies policies) {
+ int policyCount = 0;
+
+ if (policies != null) {
+ if (policies.getPolicies() != null) {
+ policyCount += policies.getPolicies().size();
+ }
+
+ if (policies.getTagPolicies() != null && policies.getTagPolicies().getPolicies() != null) {
+ policyCount += policies.getTagPolicies().getPolicies().size();
+ }
+
+ if (policies.getSecurityZones() != null) {
+ for (SecurityZoneInfo zoneInfo : policies.getSecurityZones().values()) {
+ if (zoneInfo.getPolicies() != null) {
+ policyCount += zoneInfo.getPolicies().size();
+ }
+ }
+ }
+ }
+
+ return "policyCount=" + policyCount;
+ }
+
+ private static String toSummaryStr(ServiceTags tags) {
+ int tagDefCount = 0;
+ int tagCount = 0;
+ int resourceCount = 0;
+
+ if (tags != null) {
+ if (tags.getTagDefinitions() != null) {
+ tagDefCount = tags.getTagDefinitions().size();
+ }
+
+ if (tags.getTags() != null) {
+ tagCount = tags.getTags().size();
+ }
+
+ if (tags.getServiceResources() != null) {
+ resourceCount = tags.getServiceResources().size();
+ }
+ }
+
+ return "tagDefCount=" + tagDefCount + ", tagCount" + tagCount + ", resourceCount=" + resourceCount;
+ }
+
+ private static String toSummaryStr(RangerRoles roles) {
+ int roleCount = 0;
+
+ if (roles != null) {
+ if (roles.getRangerRoles() != null) {
+ roleCount = roles.getRangerRoles().size();
+ }
+ }
+
+ return "roleCount=" + roleCount;
+ }
+
+ private static String toSummaryStr(RangerUserStore userStore) {
+ int userCount = 0;
+ int groupCount = 0;
+ int userGroupCount = 0;
+
+ if (userStore != null) {
+ if (userStore.getUserAttrMapping() != null) {
+ userCount = userStore.getUserAttrMapping().size();
+ }
+
+ if (userStore.getGroupAttrMapping() != null) {
+ groupCount = userStore.getGroupAttrMapping().size();
+ }
+
+ if (userStore.getUserGroupMapping() != null) {
+ for (Set<String> userGroups : userStore.getUserGroupMapping().values()) {
+ userGroupCount += userGroups.size();
+ }
+ }
+ }
+
+ return "users=" + userCount + ", groups=" + groupCount + ", userGroupMappings=" + userGroupCount;
+ }
+}
diff --git a/ranger-tools/testdata/test_roles_hive.json b/ranger-tools/testdata/test_roles_hive.json
new file mode 100644
index 000000000..306638f75
--- /dev/null
+++ b/ranger-tools/testdata/test_roles_hive.json
@@ -0,0 +1,235 @@
+{
+ "serviceName": "cl1_hive",
+ "roleVersion": 1,
+ "rangerRoles": [
+ {
+ "name": "r_hrt-1",
+ "description": "r_hrt-1",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_1", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_1", "isAdmin": false }
+ ],
+ "roles": [
+ ]
+ },
+ {
+ "name": "r_hrt-2",
+ "description": "r_hrt-2",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_2", "isAdmin": false },
+ { "name": "hrt_3", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_2", "isAdmin": false },
+ { "name": "g_hrt_3", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_1", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-3",
+ "description": "r_hrt-3",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_4", "isAdmin": false },
+ { "name": "hrt_5", "isAdmin": false },
+ { "name": "hrt_6", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_4", "isAdmin": false },
+ { "name": "g_hrt_5", "isAdmin": false },
+ { "name": "g_hrt_6", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_2", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-4",
+ "description": "r_hrt-4",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_7", "isAdmin": false },
+ { "name": "hrt_8", "isAdmin": false },
+ { "name": "hrt_9", "isAdmin": false },
+ { "name": "hrt_10", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_7", "isAdmin": false },
+ { "name": "g_hrt_8", "isAdmin": false },
+ { "name": "g_hrt_9", "isAdmin": false },
+ { "name": "g_hrt_10", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_3", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-5",
+ "description": "r_hrt-5",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_11", "isAdmin": false },
+ { "name": "hrt_12", "isAdmin": false },
+ { "name": "hrt_13", "isAdmin": false },
+ { "name": "hrt_14", "isAdmin": false },
+ { "name": "hrt_15", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_11", "isAdmin": false },
+ { "name": "g_hrt_12", "isAdmin": false },
+ { "name": "g_hrt_13", "isAdmin": false },
+ { "name": "g_hrt_14", "isAdmin": false },
+ { "name": "g_hrt_15", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_4", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-6",
+ "description": "r_hrt-6",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_16", "isAdmin": false },
+ { "name": "hrt_17", "isAdmin": false },
+ { "name": "hrt_18", "isAdmin": false },
+ { "name": "hrt_19", "isAdmin": false },
+ { "name": "hrt_20", "isAdmin": false },
+ { "name": "hrt_21", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_16", "isAdmin": false },
+ { "name": "g_hrt_17", "isAdmin": false },
+ { "name": "g_hrt_18", "isAdmin": false },
+ { "name": "g_hrt_19", "isAdmin": false },
+ { "name": "g_hrt_20", "isAdmin": false },
+ { "name": "g_hrt_21", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_5", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-7",
+ "description": "r_hrt-7",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_22", "isAdmin": false },
+ { "name": "hrt_23", "isAdmin": false },
+ { "name": "hrt_24", "isAdmin": false },
+ { "name": "hrt_25", "isAdmin": false },
+ { "name": "hrt_26", "isAdmin": false },
+ { "name": "hrt_27", "isAdmin": false },
+ { "name": "hrt_28", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_22", "isAdmin": false },
+ { "name": "g_hrt_23", "isAdmin": false },
+ { "name": "g_hrt_24", "isAdmin": false },
+ { "name": "g_hrt_25", "isAdmin": false },
+ { "name": "g_hrt_26", "isAdmin": false },
+ { "name": "g_hrt_27", "isAdmin": false },
+ { "name": "g_hrt_28", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_6", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-8",
+ "description": "r_hrt-8",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_29", "isAdmin": false },
+ { "name": "hrt_30", "isAdmin": false },
+ { "name": "hrt_31", "isAdmin": false },
+ { "name": "hrt_32", "isAdmin": false },
+ { "name": "hrt_33", "isAdmin": false },
+ { "name": "hrt_34", "isAdmin": false },
+ { "name": "hrt_35", "isAdmin": false },
+ { "name": "hrt_36", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_29", "isAdmin": false },
+ { "name": "g_hrt_30", "isAdmin": false },
+ { "name": "g_hrt_31", "isAdmin": false },
+ { "name": "g_hrt_32", "isAdmin": false },
+ { "name": "g_hrt_33", "isAdmin": false },
+ { "name": "g_hrt_34", "isAdmin": false },
+ { "name": "g_hrt_35", "isAdmin": false },
+ { "name": "g_hrt_36", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_7", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-9",
+ "description": "r_hrt-9",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_37", "isAdmin": false },
+ { "name": "hrt_38", "isAdmin": false },
+ { "name": "hrt_39", "isAdmin": false },
+ { "name": "hrt_40", "isAdmin": false },
+ { "name": "hrt_41", "isAdmin": false },
+ { "name": "hrt_42", "isAdmin": false },
+ { "name": "hrt_43", "isAdmin": false },
+ { "name": "hrt_44", "isAdmin": false },
+ { "name": "hrt_45", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_37", "isAdmin": false },
+ { "name": "g_hrt_38", "isAdmin": false },
+ { "name": "g_hrt_39", "isAdmin": false },
+ { "name": "g_hrt_40", "isAdmin": false },
+ { "name": "g_hrt_41", "isAdmin": false },
+ { "name": "g_hrt_42", "isAdmin": false },
+ { "name": "g_hrt_43", "isAdmin": false },
+ { "name": "g_hrt_44", "isAdmin": false },
+ { "name": "g_hrt_45", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_8", "isAdmin": false }
+ ]
+ },
+ {
+ "name": "r_hrt-10",
+ "description": "r_hrt-10",
+ "createdByUser": "admin",
+ "users": [
+ { "name": "hrt_46", "isAdmin": false },
+ { "name": "hrt_47", "isAdmin": false },
+ { "name": "hrt_48", "isAdmin": false },
+ { "name": "hrt_49", "isAdmin": false },
+ { "name": "hrt_50", "isAdmin": false },
+ { "name": "hrt_51", "isAdmin": false },
+ { "name": "hrt_52", "isAdmin": false },
+ { "name": "hrt_53", "isAdmin": false },
+ { "name": "hrt_54", "isAdmin": false },
+ { "name": "hrt_55", "isAdmin": false }
+ ],
+ "groups": [
+ { "name": "g_hrt_46", "isAdmin": false },
+ { "name": "g_hrt_47", "isAdmin": false },
+ { "name": "g_hrt_48", "isAdmin": false },
+ { "name": "g_hrt_49", "isAdmin": false },
+ { "name": "g_hrt_50", "isAdmin": false },
+ { "name": "g_hrt_51", "isAdmin": false },
+ { "name": "g_hrt_52", "isAdmin": false },
+ { "name": "g_hrt_53", "isAdmin": false },
+ { "name": "g_hrt_54", "isAdmin": false },
+ { "name": "g_hrt_55", "isAdmin": false }
+ ],
+ "roles": [
+ { "name": "r_hrt_9", "isAdmin": false }
+ ]
+ }
+ ]
+}
diff --git a/ranger-tools/testdata/test_userstore_hive.json b/ranger-tools/testdata/test_userstore_hive.json
new file mode 100644
index 000000000..931cda861
--- /dev/null
+++ b/ranger-tools/testdata/test_userstore_hive.json
@@ -0,0 +1,69 @@
+{
+ "userStoreVersion": 1,
+ "userAttrMapping": {
+ "hrt-1": { "dept": "hr", "email": "hrt-1@mycompany.org" },
+ "hrt-2": { "dept": "hr", "email": "hrt-2@mycompany.org" },
+ "hrt-3": { "dept": "hr", "email": "hrt-3@mycompany.org" },
+ "hrt-4": { "dept": "hr", "email": "hrt-4@mycompany.org" },
+ "hrt-5": { "dept": "hr", "email": "hrt-5@mycompany.org" },
+ "hrt-6": { "dept": "hr", "email": "hrt-6@mycompany.org" },
+ "hrt-7": { "dept": "hr", "email": "hrt-7@mycompany.org" },
+ "hrt-8": { "dept": "hr", "email": "hrt-8@mycompany.org" },
+ "hrt-9": { "dept": "hr", "email": "hrt-9@mycompany.org" },
+ "hrt-10": { "dept": "hr", "email": "hrt-10@mycompany.org" },
+ "hrt-11": { "dept": "hr", "email": "hrt-11@mycompany.org" },
+ "hrt-12": { "dept": "hr", "email": "hrt-12@mycompany.org" },
+ "hrt-13": { "dept": "hr", "email": "hrt-13@mycompany.org" },
+ "hrt-14": { "dept": "hr", "email": "hrt-14@mycompany.org" },
+ "hrt-15": { "dept": "hr", "email": "hrt-15@mycompany.org" },
+ "hrt-16": { "dept": "hr", "email": "hrt-16@mycompany.org" },
+ "hrt-17": { "dept": "hr", "email": "hrt-17@mycompany.org" },
+ "hrt-18": { "dept": "hr", "email": "hrt-18@mycompany.org" },
+ "hrt-19": { "dept": "hr", "email": "hrt-19@mycompany.org" },
+ "hrt-20": { "dept": "hr", "email": "hrt-20@mycompany.org" }
+ },
+ "groupAttrMapping": {
+ "g_hrt-1": { "id": "1", "email": "g_hrt-1@mycompany.org" },
+ "g_hrt-2": { "id": "2", "email": "g_hrt-2@mycompany.org" },
+ "g_hrt-3": { "id": "3", "email": "g_hrt-3@mycompany.org" },
+ "g_hrt-4": { "id": "4", "email": "g_hrt-4@mycompany.org" },
+ "g_hrt-5": { "id": "5", "email": "g_hrt-5@mycompany.org" },
+ "g_hrt-6": { "id": "6", "email": "g_hrt-6@mycompany.org" },
+ "g_hrt-7": { "id": "7", "email": "g_hrt-7@mycompany.org" },
+ "g_hrt-8": { "id": "8", "email": "g_hrt-8@mycompany.org" },
+ "g_hrt-9": { "id": "9", "email": "g_hrt-9@mycompany.org" },
+ "g_hrt-10": { "id": "10", "email": "g_hrt-10@mycompany.org" },
+ "g_hrt-11": { "id": "11", "email": "g_hrt-11@mycompany.org" },
+ "g_hrt-12": { "id": "12", "email": "g_hrt-12@mycompany.org" },
+ "g_hrt-13": { "id": "13", "email": "g_hrt-13@mycompany.org" },
+ "g_hrt-14": { "id": "14", "email": "g_hrt-14@mycompany.org" },
+ "g_hrt-15": { "id": "15", "email": "g_hrt-15@mycompany.org" },
+ "g_hrt-16": { "id": "16", "email": "g_hrt-16@mycompany.org" },
+ "g_hrt-17": { "id": "17", "email": "g_hrt-17@mycompany.org" },
+ "g_hrt-18": { "id": "18", "email": "g_hrt-18@mycompany.org" },
+ "g_hrt-19": { "id": "19", "email": "g_hrt-19@mycompany.org" },
+ "g_hrt-20": { "id": "20", "email": "g_hrt-20@mycompany.org" }
+ },
+ "userGroupMapping": {
+ "hrt-1": [ "g_hrt-1" ],
+ "hrt-2": [ "g_hrt-2" ],
+ "hrt-3": [ "g_hrt-3" ],
+ "hrt-4": [ "g_hrt-4" ],
+ "hrt-5": [ "g_hrt-5" ],
+ "hrt-6": [ "g_hrt-6" ],
+ "hrt-7": [ "g_hrt-7" ],
+ "hrt-8": [ "g_hrt-8" ],
+ "hrt-9": [ "g_hrt-9" ],
+ "hrt-10": [ "g_hrt-10" ],
+ "hrt-11": [ "g_hrt-11" ],
+ "hrt-12": [ "g_hrt-12" ],
+ "hrt-13": [ "g_hrt-13" ],
+ "hrt-14": [ "g_hrt-14" ],
+ "hrt-15": [ "g_hrt-15" ],
+ "hrt-16": [ "g_hrt-16" ],
+ "hrt-17": [ "g_hrt-17" ],
+ "hrt-18": [ "g_hrt-18" ],
+ "hrt-19": [ "g_hrt-19" ],
+ "hrt-20": [ "g_hrt-20" ]
+ }
+}