You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2023/03/03 18:34:34 UTC
[ranger] branch master updated: RANGER-3903: performance improvement in RangerPolicyDeltaUtil.applyDeltas()
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new d35382101 RANGER-3903: performance improvement in RangerPolicyDeltaUtil.applyDeltas()
d35382101 is described below
commit d35382101c719870bb72f8b413fa6b97a1aa2552
Author: Ramachandran Krishnan <ra...@gmail.com>
AuthorDate: Fri Feb 24 10:29:59 2023 +0530
RANGER-3903: performance improvement in RangerPolicyDeltaUtil.applyDeltas()
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../ranger/plugin/util/RangerPolicyDeltaUtil.java | 111 ++++++++-------------
1 file changed, 42 insertions(+), 69 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
index e9223fe69..86b18aace 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java
@@ -21,6 +21,7 @@ package org.apache.ranger.plugin.util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
@@ -29,8 +30,9 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Iterator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class RangerPolicyDeltaUtil {
@@ -44,94 +46,69 @@ public class RangerPolicyDeltaUtil {
}
List<RangerPolicy> ret;
-
- RangerPerfTracer perf = null;
+ RangerPerfTracer perf = null;
if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()");
}
- boolean hasExpectedServiceType = false;
-
if (CollectionUtils.isNotEmpty(deltas)) {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType + ")");
}
- for (RangerPolicyDelta delta : deltas) {
- if (serviceType.equals(delta.getServiceType())) {
- hasExpectedServiceType = true;
- break;
- } else if (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && !delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
- LOG.warn("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing");
- }
- }
-
- if (hasExpectedServiceType) {
- ret = new ArrayList<>(policies);
+ Map<Long, RangerPolicy> retMap = new HashMap<>();
- for (RangerPolicyDelta delta : deltas) {
- if (!serviceType.equals(delta.getServiceType())) {
- continue;
- }
+ for (RangerPolicy policy : policies) {
+ retMap.put(policy.getId(), policy);
+ }
- int changeType = delta.getChangeType();
+ for (RangerPolicyDelta delta : deltas) {
+ if (!StringUtils.equals(serviceType, delta.getServiceType()) || delta.getPolicyId() == null) {
+ continue;
+ }
- if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
- Long policyId = delta.getPolicyId();
+ int changeType = delta.getChangeType();
- if (policyId == null) {
- continue;
- }
+ if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta");
- List<RangerPolicy> deletedPolicies = new ArrayList<>();
+ continue;
+ }
- Iterator<RangerPolicy> iter = ret.iterator();
+ Long policyId = delta.getPolicyId();
+ RangerPolicy deletedPolicy = retMap.remove(policyId);
- while (iter.hasNext()) {
- RangerPolicy policy = iter.next();
- if (policyId.equals(policy.getId()) && (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE)) {
- deletedPolicies.add(policy);
- iter.remove();
- }
+ switch(changeType) {
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: {
+ if (deletedPolicy != null) {
+ LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + deletedPolicy);
}
-
- switch(changeType) {
- case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: {
- if (CollectionUtils.isNotEmpty(deletedPolicies)) {
- LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + Arrays.toString(deletedPolicies.toArray()));
- }
- break;
- }
- case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: {
- if (CollectionUtils.isEmpty(deletedPolicies) || deletedPolicies.size() > 1) {
- LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray()));
- }
- break;
- }
- case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: {
- if (CollectionUtils.isEmpty(deletedPolicies) || deletedPolicies.size() > 1) {
- LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray()));
- }
- break;
- }
- default:
- break;
+ break;
+ }
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: {
+ if (deletedPolicy == null) {
+ LOG.warn("Unexpected: found no existing policy for CHANGE_TYPE_POLICY_UPDATE: " + deletedPolicy);
}
-
- if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
- ret.add(delta.getPolicy());
+ break;
+ }
+ case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: {
+ if (deletedPolicy == null) {
+ LOG.warn("Unexpected: found no existing policy for CHANGE_TYPE_POLICY_DELETE: " + deletedPolicy);
}
- } else {
- LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta");
+ break;
}
+ default:
+ break;
}
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("applyDeltas - none of the deltas is for " + serviceType + ")");
+
+ if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
+ retMap.put(policyId, delta.getPolicy());
}
- ret = policies;
}
+
+ ret = new ArrayList<>(retMap.values());
+ ret.sort(RangerPolicy.POLICY_ID_COMPARATOR);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas called with empty deltas. Will return policies without change");
@@ -139,10 +116,6 @@ public class RangerPolicyDeltaUtil {
ret = policies;
}
- if (CollectionUtils.isNotEmpty(deltas) && hasExpectedServiceType && CollectionUtils.isNotEmpty(ret)) {
- ret.sort(RangerPolicy.POLICY_ID_COMPARATOR);
- }
-
RangerPerfTracer.log(perf);
if (LOG.isDebugEnabled()) {