You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2025/02/11 20:52:51 UTC
(ranger) branch master updated: RANGER-5129: Enhance Dataset Filtering by Validity Schedule and Expiration Date Range
This is an automated email from the ASF dual-hosted git repository.
rmani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new 8c543062f5 RANGER-5129: Enhance Dataset Filtering by Validity Schedule and Expiration Date Range
8c543062f5 is described below
commit 8c543062f50d11c453a8dccfb177a98b4f0d42ff
Author: Radhika Kundam <ra...@gmail.com>
AuthorDate: Mon Feb 10 14:03:55 2025 -0800
RANGER-5129: Enhance Dataset Filtering by Validity Schedule and Expiration Date Range
Signed-off-by: Ramesh Mani <rm...@apache.org>
---
.../RangerValidityScheduleEvaluator.java | 2 +-
.../apache/ranger/plugin/util/SearchFilter.java | 4 +
.../java/org/apache/ranger/biz/GdsDBStore.java | 70 ++++++++++
.../org/apache/ranger/common/RangerSearchUtil.java | 3 +
.../java/org/apache/ranger/rest/TestGdsREST.java | 149 ++++++++++++++++++++-
5 files changed, 226 insertions(+), 2 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
index 027c2140c6..5dac4d9f93 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerValidityScheduleEvaluator.java
@@ -50,7 +50,7 @@ public class RangerValidityScheduleEvaluator {
private static final TimeZone defaultTZ = TimeZone.getDefault();
- private static final ThreadLocal<DateFormat> DATE_FORMATTER = ThreadLocal.withInitial(() -> new SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION));
+ public static final ThreadLocal<DateFormat> DATE_FORMATTER = ThreadLocal.withInitial(() -> new SimpleDateFormat(RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION));
private final Date startTime;
private final Date endTime;
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
index a260a87828..5a5556a2f2 100755
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
@@ -140,6 +140,10 @@ public class SearchFilter {
public static final String IS_DISTINCT = "isDistinct"; // search, sort
public static final String RETRIEVE_ALL_PAGES = "retrieveAllPages"; // search
public static final String SHARED_WITH_ME = "sharedWithMe"; // search
+ public static final String VALIDITY_EXPIRY_START = "validityExpiryStart"; // search
+ public static final String VALIDITY_EXPIRY_END = "validityExpiryEnd"; // search
+ public static final String VALIDITY_TIME_ZONE = "timeZone";
+ public static final String DEFAULT_TIME_ZONE = "GMT";
private Map<String, String> params;
private Map<String, Object[]> multiValueParams;
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
index 312a67a28f..b1431cf71d 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
@@ -64,6 +64,10 @@
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
import org.apache.ranger.plugin.model.RangerPolicyDelta;
import org.apache.ranger.plugin.model.RangerPrincipal.PrincipalType;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import org.apache.ranger.plugin.model.validation.RangerValidityScheduleValidator;
+import org.apache.ranger.plugin.model.validation.ValidationFailureDetails;
+import org.apache.ranger.plugin.policyevaluator.RangerValidityScheduleEvaluator;
import org.apache.ranger.plugin.store.AbstractGdsStore;
import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServiceStore;
@@ -91,10 +95,12 @@
import javax.annotation.PostConstruct;
+import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -107,6 +113,7 @@
import java.util.stream.Collectors;
import static org.apache.ranger.db.XXGlobalStateDao.RANGER_GLOBAL_STATE_NAME_GDS;
+import static org.apache.ranger.plugin.policyevaluator.RangerValidityScheduleEvaluator.DATE_FORMATTER;
import static org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_GDS_NAME;
@Component
@@ -1786,9 +1793,72 @@ private List<RangerDataset> fetchDatasetsBySearchCriteria(SearchFilter filter) {
}
}
+ filterDatasetsByValidityExpiration(filter, datasets);
+
return datasets;
}
+ public void filterDatasetsByValidityExpiration(SearchFilter filter, List<RangerDataset> datasets) {
+ LOG.debug("==> filterDatasetsByValidityExpiration({}, {})", filter, datasets);
+ String validityCheckStart = filter.getParam(SearchFilter.VALIDITY_EXPIRY_START);
+ String validityCheckEnd = filter.getParam(SearchFilter.VALIDITY_EXPIRY_END);
+ String validityTimeZone = filter.getParam(SearchFilter.VALIDITY_TIME_ZONE);
+ RangerValiditySchedule validityCheckFilter = new RangerValiditySchedule(validityCheckStart, validityCheckEnd, validityTimeZone, null);
+ List<ValidationFailureDetails> failures = new ArrayList<>();
+ RangerValiditySchedule validatedValidityCheckFilter = validateValidityCheckFilter(validityCheckFilter, failures);
+
+ if (validatedValidityCheckFilter != null) {
+ RangerValidityScheduleEvaluator validityScheduleEvaluator = new RangerValidityScheduleEvaluator(validatedValidityCheckFilter);
+ datasets.removeIf(dataset -> !isDatasetExpiring(dataset, validityScheduleEvaluator, failures));
+ }
+
+ if (CollectionUtils.isNotEmpty(failures)) {
+ throw restErrorUtil.createRESTException("Error in finding datasets expiring between '" + validityCheckStart + "' and '" + validityCheckEnd + "': " + failures);
+ }
+ LOG.debug("==> filterDatasetsByValidityExpiration({}, {})", filter, datasets);
+ }
+
+ private boolean isDatasetExpiring(RangerDataset dataset, RangerValidityScheduleEvaluator validityScheduleEvaluator, List<ValidationFailureDetails> failures) {
+ if (dataset.getValiditySchedule() == null) {
+ return false;
+ }
+ String datasetValidityScheduleEndTime = dataset.getValiditySchedule().getEndTime();
+ if (StringUtils.isEmpty(datasetValidityScheduleEndTime)) {
+ return false;
+ }
+
+ try {
+ Date datasetExpiryTime = DATE_FORMATTER.get().parse(datasetValidityScheduleEndTime);
+ return validityScheduleEvaluator.isApplicable(datasetExpiryTime.getTime());
+ } catch (ParseException pe) {
+ failures.add(new ValidationFailureDetails(0, "endTime", "", false, true, false, "Error parsing endTime:" + datasetValidityScheduleEndTime));
+ }
+ return false;
+ }
+
+ private RangerValiditySchedule validateValidityCheckFilter(RangerValiditySchedule validityCheckFilter, List<ValidationFailureDetails> failures) {
+ String startTime = validityCheckFilter.getStartTime();
+ String endTime = validityCheckFilter.getEndTime();
+ String timeZone = validityCheckFilter.getTimeZone();
+
+ if (StringUtils.isEmpty(startTime) && StringUtils.isEmpty(endTime)) {
+ return null;
+ }
+
+ if (StringUtils.isEmpty(startTime) || StringUtils.isEmpty(endTime)) {
+ failures.add(new ValidationFailureDetails(0, "startTime,endTime", "", true, true, false, "empty values"));
+ return null;
+ }
+
+ if (StringUtils.isEmpty(timeZone)) {
+ validityCheckFilter.setTimeZone(SearchFilter.DEFAULT_TIME_ZONE);
+ }
+
+ RangerValidityScheduleValidator validator = new RangerValidityScheduleValidator(validityCheckFilter);
+
+ return validator.validate(failures);
+ }
+
private PList<RangerDataset> applyPaginataionAndSorting(List<RangerDataset> datasets, SearchFilter filter) {
int maxRows = filter.getMaxRows();
int startIndex = filter.getStartIndex();
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
index 6c93d71040..6833048e66 100755
--- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
@@ -150,6 +150,9 @@ public SearchFilter getSearchFilter(@Nonnull HttpServletRequest request, List<So
ret.setParam(SearchFilter.UPDATE_TIME_END, request.getParameter(SearchFilter.UPDATE_TIME_END));
ret.setParam(SearchFilter.RESOURCE_CONTAINS, request.getParameter(SearchFilter.RESOURCE_CONTAINS));
ret.setParam(SearchFilter.SHARED_WITH_ME, request.getParameter(SearchFilter.SHARED_WITH_ME));
+ ret.setParam(SearchFilter.VALIDITY_EXPIRY_START, request.getParameter(SearchFilter.VALIDITY_EXPIRY_START));
+ ret.setParam(SearchFilter.VALIDITY_EXPIRY_END, request.getParameter(SearchFilter.VALIDITY_EXPIRY_END));
+ ret.setParam(SearchFilter.VALIDITY_TIME_ZONE, request.getParameter(SearchFilter.VALIDITY_TIME_ZONE));
extractCommonCriteriasForFilter(request, ret, sortFields);
diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
index 11df009a46..143cfe9a90 100644
--- a/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
+++ b/security-admin/src/test/java/org/apache/ranger/rest/TestGdsREST.java
@@ -16,11 +16,15 @@
*/
package org.apache.ranger.rest;
+import org.apache.ranger.biz.GdsDBStore;
+import org.apache.ranger.common.RESTErrorUtil;
import org.apache.ranger.common.RangerSearchUtil;
import org.apache.ranger.plugin.model.RangerGds;
import org.apache.ranger.plugin.model.RangerGrant;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerPrincipal;
+import org.apache.ranger.plugin.model.RangerValiditySchedule;
+import org.apache.ranger.plugin.util.SearchFilter;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -31,7 +35,10 @@
import org.mockito.junit.MockitoJUnitRunner;
import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -42,6 +49,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
@@ -52,7 +60,11 @@ public class TestGdsREST {
@Mock
RangerSearchUtil searchUtil;
@InjectMocks
- private GdsREST gdsREST = new GdsREST();
+ private GdsREST gdsREST;
+ @InjectMocks
+ GdsDBStore gdsDBStore;
+ @Mock
+ RESTErrorUtil restErrorUtil;
@Test
public void testAddDataSetGrants() {
@@ -207,6 +219,116 @@ public void testGetDataSetGrantsByPrincipalAndAccessType() {
assertTrue("Grants for Principals: " + Arrays.toString(requestedPrincipals) + " and AccessTypes: " + Arrays.toString(nonexistentRequestedAccessTypes) + " should be empty", updatedPolicyItemsByAccessType.isEmpty());
}
+ @Test
+ public void testSearchDataSetsByValidityPeriod() {
+ List<RangerGds.RangerDataset> rangerDatasets = new ArrayList<>();
+
+ RangerGds.RangerDataset rangerDataset1 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset1, -5, -1);
+
+ RangerGds.RangerDataset rangerDataset2 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset2, -5, 5);
+
+ RangerGds.RangerDataset rangerDataset3 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset3, -10, 2);
+
+ RangerGds.RangerDataset rangerDataset4 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset4, -2, 15);
+
+ RangerGds.RangerDataset rangerDataset5 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset5, 5, 15);
+
+ RangerGds.RangerDataset rangerDataset6 = createRangerDataSet();
+ updateDatasetValiditySchedule(rangerDataset6, -15, -5);
+
+ rangerDatasets.addAll(Arrays.asList(rangerDataset1, rangerDataset2, rangerDataset3, rangerDataset4, rangerDataset5, rangerDataset6));
+ List<RangerGds.RangerDataset> actualDatasets = new ArrayList<>(rangerDatasets);
+
+ SearchFilter filter = new SearchFilter();
+ filter.setParam(SearchFilter.VALIDITY_TIME_ZONE, SearchFilter.DEFAULT_TIME_ZONE);
+
+ //ValiditySchedule Filter criteria-1
+ String startTime = getFormattedDateString(-10);
+ String endTime = getFormattedDateString(-2);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ List<RangerGds.RangerDataset> expectedDatasets = Arrays.asList(rangerDataset6);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + " and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " + startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-2
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getFormattedDateString(-4);
+ endTime = getFormattedDateString(20);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ expectedDatasets = Arrays.asList(rangerDataset1, rangerDataset2, rangerDataset3, rangerDataset4, rangerDataset5);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + " and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " + startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-3
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getFormattedDateString(-15);
+ endTime = getFormattedDateString(0);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ expectedDatasets = Arrays.asList(rangerDataset1, rangerDataset6);
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch between " + startTime + " and " + endTime, expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned for expiry between " + startTime + " and " + endTime,
+ actualDatasets.containsAll(expectedDatasets));
+
+ //ValiditySchedule Filter criteria-4 with invalid date
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ startTime = getInvalidDateString(-5);
+ endTime = getFormattedDateString(16);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_START, startTime);
+ filter.setParam(SearchFilter.VALIDITY_EXPIRY_END, endTime);
+
+ Mockito.when(restErrorUtil.createRESTException(Mockito.anyString())).thenThrow(new WebApplicationException());
+
+ assertThrows(WebApplicationException.class, () -> {
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+ });
+
+ //ValiditySchedule Filter criteria-5 without start and end time
+ actualDatasets.clear();
+ actualDatasets.addAll(rangerDatasets);
+
+ filter.removeParam(SearchFilter.VALIDITY_EXPIRY_START);
+ filter.removeParam(SearchFilter.VALIDITY_EXPIRY_END);
+
+ expectedDatasets = rangerDatasets;
+
+ gdsDBStore.filterDatasetsByValidityExpiration(filter, actualDatasets);
+
+ assertEquals("Datasets expiry count mismatch with empty start and end time", expectedDatasets.size(), actualDatasets.size());
+
+ assertTrue("Mismatch in datasets returned with empty expiry time range", actualDatasets.containsAll(expectedDatasets));
+ }
+
private RangerGds.RangerDataset createRangerDataSet() {
long id = new Random().nextInt(100);
RangerGds.RangerDataset dataset = new RangerGds.RangerDataset();
@@ -217,6 +339,31 @@ private RangerGds.RangerDataset createRangerDataSet() {
return dataset;
}
+ private RangerGds.RangerDataset updateDatasetValiditySchedule(RangerGds.RangerDataset dataset, int pastDaysToStart, int futureDaysToEnd) {
+ String start = getFormattedDateString(pastDaysToStart);
+ String end = getFormattedDateString(futureDaysToEnd);
+ String timezone = SearchFilter.DEFAULT_TIME_ZONE;
+ RangerValiditySchedule validitySchedule = new RangerValiditySchedule(start, end, timezone, null);
+ dataset.setValiditySchedule(validitySchedule);
+
+ return dataset;
+ }
+
+ private String getFormattedDateString(int days) {
+ return getDateString(days, RangerValiditySchedule.VALIDITY_SCHEDULE_DATE_STRING_SPECIFICATION);
+ }
+
+ private String getInvalidDateString(int days) {
+ return getDateString(days, "yyyy/MM/dd");
+ }
+
+ private String getDateString(int days, String pattern) {
+ DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern(pattern);
+ LocalDateTime current = LocalDateTime.now();
+
+ return current.plusDays(days).format(dateFormatter);
+ }
+
private RangerPolicy createPolicyForDataSet(RangerGds.RangerDataset dataset) {
RangerPolicy policy = new RangerPolicy();
policy.setName("DATASET: " + dataset.getName() + "@" + System.currentTimeMillis());