You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2025/02/10 20:29:29 UTC
(ranger) branch master updated: RANGER-5111: Enhance Datashare Summary API to Include Labels, Keywords, and Share Counts for Actionable Filters
This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new c334ba80a4 RANGER-5111: Enhance Datashare Summary API to Include Labels, Keywords, and Share Counts for Actionable Filters
c334ba80a4 is described below
commit c334ba80a4bb38653a00f61cd1bec0927194a018
Author: Radhika Kundam <ra...@gmail.com>
AuthorDate: Wed Jan 22 16:53:35 2025 -0800
RANGER-5111: Enhance Datashare Summary API to Include Labels, Keywords, and Share Counts for Actionable Filters
Signed-off-by: Ramesh Mani <rm...@apache.org>
---
.../org/apache/ranger/plugin/model/RangerGds.java | 51 ++++++++++++--
.../java/org/apache/ranger/plugin/store/PList.java | 22 +++++++
.../java/org/apache/ranger/biz/GdsDBStore.java | 77 ++++++++++++++++++++--
.../main/java/org/apache/ranger/rest/GdsREST.java | 39 +++++++++--
4 files changed, 173 insertions(+), 16 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
index 260ebc0a8b..6bdc02691a 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerGds.java
@@ -26,11 +26,13 @@
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemRowFilterInfo;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType;
+import org.apache.ranger.plugin.store.PList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -167,8 +169,8 @@ public StringBuilder toString(StringBuilder sb) {
.append("acl={").append(acl).append("} ")
.append("validitySchedule={").append(validitySchedule).append("} ")
.append("termsOfUse={").append(termsOfUse).append("} ")
- .append("labels={").append(validitySchedule).append("} ")
- .append("keywords={").append(termsOfUse).append("} ")
+ .append("labels={").append(labels).append("} ")
+ .append("keywords={").append(keywords).append("} ")
.append("}");
return sb;
@@ -860,9 +862,48 @@ public StringBuilder toString(StringBuilder sb) {
.append("aclPrincipalsCount={").append(aclPrincipalsCount).append("} ")
.append("totalResourceCount={").append(totalResourceCount).append("} ")
.append("dataShares={").append(dataShares).append("} ")
- .append("validitySchedule={").append(totalResourceCount).append("} ")
- .append("labels={").append(totalResourceCount).append("} ")
- .append("keywords={").append(totalResourceCount).append("} ")
+ .append("validitySchedule={").append(validitySchedule).append("} ")
+ .append("labels={").append(labels).append("} ")
+ .append("keywords={").append(keywords).append("} ")
+ .append("}");
+
+ return sb;
+ }
+ }
+
+ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class DatasetsSummary extends PList<DatasetSummary> {
+ private static final long serialVersionUID = 1L;
+ private Map<String, Map<String, Integer>> additionalInfo;
+
+ public DatasetsSummary() {
+ super();
+ }
+
+ public DatasetsSummary(PList<DatasetSummary> datasetSummary, Map<String, Map<String, Integer>> additionalInfo) {
+ super(datasetSummary);
+ this.additionalInfo = (additionalInfo != null) ? additionalInfo : Collections.emptyMap();
+ }
+
+ public Map<String, Map<String, Integer>> getAdditionalInfo() {
+ return additionalInfo;
+ }
+
+ public void setAdditionalInfo(Map<String, Map<String, Integer>> additionalInfo) {
+ this.additionalInfo = additionalInfo;
+ }
+
+ @Override
+ public String toString() {
+ return toString(new StringBuilder()).toString();
+ }
+
+ public StringBuilder toString(StringBuilder sb) {
+ sb.append("DatasetsSummary={")
+ .append("list={").append(this.list).append("} ")
+ .append("additionalInfo={").append(additionalInfo).append("} ")
.append("}");
return sb;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
index 4bf2ee02fe..1a1b16a5bf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
@@ -19,6 +19,8 @@
package org.apache.ranger.plugin.store;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
public class PList<T> implements java.io.Serializable {
@@ -75,6 +77,26 @@ public PList(List<T> list, int startIndex, int pageSize, long totalCount, int re
this.sortBy = sortBy;
}
+ public PList(PList<T> other) {
+ if (other != null) {
+ this.list = other.getList() != null ? new ArrayList<>(other.getList()) : Collections.emptyList();
+ this.startIndex = other.getStartIndex();
+ this.pageSize = other.getPageSize();
+ this.totalCount = other.getTotalCount();
+ this.resultSize = other.getResultSize();
+ this.sortType = other.getSortType();
+ this.sortBy = other.getSortBy();
+ } else {
+ this.list = Collections.emptyList();
+ this.startIndex = 0;
+ this.pageSize = 0;
+ this.totalCount = 0;
+ this.resultSize = 0;
+ this.sortType = null;
+ this.sortBy = null;
+ }
+ }
+
public int getListSize() {
return list == null ? 0 : list.size();
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
index 7916f08188..312a67a28f 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
@@ -48,6 +48,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
+import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.GdsPermission;
import org.apache.ranger.plugin.model.RangerGds.GdsShareStatus;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
@@ -120,6 +121,9 @@ public class GdsDBStore extends AbstractGdsStore {
public static final String NOT_AUTHORIZED_TO_VIEW_PROJECT_POLICIES = "User is not authorized to view policies for this dataset";
public static final String GDS_POLICY_NAME_TIMESTAMP_SEP = "@";
+ public static final String LABELS = "labelCounts";
+ public static final String KEYWORDS = "keywordCounts";
+
private static final Set<Integer> SHARE_STATUS_AGR = new HashSet<>(Arrays.asList(GdsShareStatus.ACTIVE.ordinal(), GdsShareStatus.GRANTED.ordinal(), GdsShareStatus.REQUESTED.ordinal()));
@Autowired
@@ -1370,19 +1374,68 @@ public ServiceGdsInfo getGdsInfoIfUpdated(String serviceName, Long lastKnownVers
}
public PList<DatasetSummary> getDatasetSummary(SearchFilter filter) {
- LOG.debug("==> getDatasetSummary({})", filter);
+ return getDatasetSummary(filter, false);
+ }
- PList<RangerDataset> datasets = getUnscrubbedDatasets(filter);
- List<DatasetSummary> datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
- PList<DatasetSummary> ret = new PList<>(datasetSummary, datasets.getStartIndex(), datasets.getPageSize(), datasets.getTotalCount(), datasets.getResultSize(), datasets.getSortType(), datasets.getSortBy());
+ public DatasetsSummary getEnhancedDatasetSummary(SearchFilter filter) {
+ return getDatasetSummary(filter, true);
+ }
- ret.setQueryTimeMS(datasets.getQueryTimeMS());
+ public DatasetsSummary getDatasetSummary(SearchFilter filter, boolean includeAdditionalInfo) {
+ LOG.debug("==> getDatasetSummary({}, {})", filter, includeAdditionalInfo);
- LOG.debug("<== getDatasetSummary({}): ret={}", filter, ret);
+ PList<RangerDataset> datasets;
+ Map<String, Map<String, Integer>> additionalInfo = null;
+
+ if (includeAdditionalInfo) {
+ List<RangerDataset> datasetsMatchingCriteria = fetchDatasetsBySearchCriteria(filter);
+ additionalInfo = buildAdditionalInfoForDatasets(datasetsMatchingCriteria);
+ datasets = applyPaginataionAndSorting(datasetsMatchingCriteria, filter);
+ } else {
+ datasets = getUnscrubbedDatasets(filter);
+ }
+
+ List<DatasetSummary> datasetSummary = toDatasetSummary(datasets.getList(), getGdsPermissionFromFilter(filter));
+ PList<DatasetSummary> paginatedDatasetSummary = createdPaginatedDatasetSummary(datasets, datasetSummary);
+ DatasetsSummary ret = new DatasetsSummary(paginatedDatasetSummary, additionalInfo);
+
+ LOG.debug("<== getDatasetSummary({}, {}): ret={}", filter, includeAdditionalInfo, ret);
return ret;
}
+ private Map<String, Map<String, Integer>> buildAdditionalInfoForDatasets(List<RangerDataset> datasets) {
+ Map<String, Map<String, Integer>> additionalInfo = new HashMap<>();
+ for (RangerDataset dataset : datasets) {
+ updateAdditionalInfo(LABELS, dataset.getLabels(), additionalInfo);
+ updateAdditionalInfo(KEYWORDS, dataset.getKeywords(), additionalInfo);
+ }
+ return additionalInfo;
+ }
+
+ private void updateAdditionalInfo(String field, List<String> fieldValues, Map<String, Map<String, Integer>> additionalInfo) {
+ if (CollectionUtils.isNotEmpty(fieldValues)) {
+ Map<String, Integer> aggregatedFieldMap = additionalInfo.computeIfAbsent(field, key -> new HashMap<>());
+ for (String value : fieldValues) {
+ aggregatedFieldMap.put(value, aggregatedFieldMap.getOrDefault(value, 0) + 1);
+ }
+ }
+ }
+
+ private PList<DatasetSummary> createdPaginatedDatasetSummary(PList<RangerDataset> datasets, List<DatasetSummary> datasetSummary) {
+ PList<DatasetSummary> paginatedDatasetSummary = new PList<>(
+ datasetSummary,
+ datasets.getStartIndex(),
+ datasets.getPageSize(),
+ datasets.getTotalCount(),
+ datasets.getResultSize(),
+ datasets.getSortType(),
+ datasets.getSortBy());
+
+ paginatedDatasetSummary.setQueryTimeMS(datasets.getQueryTimeMS());
+ return paginatedDatasetSummary;
+ }
+
public PList<DataShareSummary> getDataShareSummary(SearchFilter filter) {
LOG.debug("==> getDataShareSummary({})", filter);
@@ -1698,6 +1751,12 @@ private PList<RangerProject> getUnscrubbedProjects(SearchFilter filter) {
}
private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
+ List<RangerDataset> datasets = fetchDatasetsBySearchCriteria(filter);
+
+ return applyPaginataionAndSorting(datasets, filter);
+ }
+
+ private List<RangerDataset> fetchDatasetsBySearchCriteria(SearchFilter filter) {
filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");
GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
@@ -1727,10 +1786,14 @@ private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
}
}
+ return datasets;
+ }
+
+ private PList<RangerDataset> applyPaginataionAndSorting(List<RangerDataset> datasets, SearchFilter filter) {
int maxRows = filter.getMaxRows();
int startIndex = filter.getStartIndex();
- return getPList(datasets, startIndex, maxRows, result.getSortBy(), result.getSortType());
+ return getPList(datasets, startIndex, maxRows, filter.getSortBy(), filter.getSortType());
}
private PList<RangerDataShare> getUnscrubbedDataShares(SearchFilter filter) {
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
index 0d3ef3d767..05bc4fd17b 100755
--- a/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/GdsREST.java
@@ -36,6 +36,7 @@
import org.apache.ranger.plugin.model.RangerGds.DataShareInDatasetSummary;
import org.apache.ranger.plugin.model.RangerGds.DataShareSummary;
import org.apache.ranger.plugin.model.RangerGds.DatasetSummary;
+import org.apache.ranger.plugin.model.RangerGds.DatasetsSummary;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShare;
import org.apache.ranger.plugin.model.RangerGds.RangerDataShareInDataset;
import org.apache.ranger.plugin.model.RangerGds.RangerDataset;
@@ -449,13 +450,13 @@ public PList<DatasetSummary> getDatasetSummary(@Context HttpServletRequest reque
LOG.debug("==> GdsREST.getDatasetSummary()");
PList<DatasetSummary> ret;
- RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()");
- SearchFilter filter = null;
+ RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getDatasetSummary()");
+ SearchFilter filter = null;
try {
filter = searchUtil.getSearchFilter(request, datasetService.sortFields);
- ret = gdsStore.getDatasetSummary(filter);
+ ret = gdsStore.getDatasetSummary(filter);
} catch (WebApplicationException we) {
throw we;
} catch (Throwable ex) {
@@ -466,7 +467,37 @@ public PList<DatasetSummary> getDatasetSummary(@Context HttpServletRequest reque
RangerPerfTracer.log(perf);
}
- LOG.debug("<== GdsREST.getDatasetSummary(): {}", ret);
+ LOG.debug("<== GdsREST.getDatasetSummary()");
+
+ return ret;
+ }
+
+ @GET
+ @Path("/dataset/enhancedsummary")
+ @Produces("application/json")
+ @PreAuthorize("@rangerPreAuthSecurityHandler.isAPIAccessible(\"" + RangerAPIList.GET_DATASET_SUMMARY + "\")")
+ public DatasetsSummary getEnhancedDatasetSummary(@Context HttpServletRequest request) {
+ LOG.debug("==> GdsREST.getEnhancedDatasetSummary()");
+
+ DatasetsSummary ret;
+ RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "GdsREST.getEnhancedDatasetSummary()");
+ SearchFilter filter = null;
+
+ try {
+ filter = searchUtil.getSearchFilter(request, datasetService.sortFields);
+
+ ret = gdsStore.getEnhancedDatasetSummary(filter);
+ } catch (WebApplicationException we) {
+ throw we;
+ } catch (Throwable ex) {
+ LOG.error("getEnhancedDatasetSummary({}) failed", filter, ex);
+
+ throw restErrorUtil.createRESTException(ex.getMessage());
+ } finally {
+ RangerPerfTracer.log(perf);
+ }
+
+ LOG.debug("<== GdsREST.getEnhancedDatasetSummary()");
return ret;
}