You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2023/12/06 16:42:16 UTC
(ranger) branch RANGER-3923 updated: RANGER-4579: updated GDS REST APIs to support retrieval of datasets/projects shared with the caller
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch RANGER-3923
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/RANGER-3923 by this push:
new e013def2a RANGER-4579: updated GDS REST APIs to support retrieval of datasets/projects shared with the caller
e013def2a is described below
commit e013def2adf494231130c605f95f9c2e267cf8e6
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Mon Dec 4 15:21:41 2023 -0800
RANGER-4579: updated GDS REST APIs to support retrieval of datasets/projects shared with the caller
---
.../plugin/policyengine/gds/GdsPolicyEngine.java | 128 ++++++++-----
.../apache/ranger/plugin/util/SearchFilter.java | 3 +-
.../policyengine/gds/TestGdsPolicyEngine.java | 27 ++-
.../gds/test_gds_policy_engine_hive.json | 60 ++++--
.../java/org/apache/ranger/biz/GdsDBStore.java | 109 +++++++----
.../org/apache/ranger/biz/GdsPolicyAdminCache.java | 209 +++++++++++++++++++++
.../org/apache/ranger/common/RangerSearchUtil.java | 1 +
.../apache/ranger/common/ServiceGdsInfoCache.java | 36 +---
.../org/apache/ranger/util/RangerAdminCache.java | 97 ++++++++++
.../ranger/util/RangerCacheDBValueLoader.java | 62 ------
10 files changed, 533 insertions(+), 199 deletions(-)
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
index 541f0754e..dd1184a0b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/gds/GdsPolicyEngine.java
@@ -123,24 +123,24 @@ public class GdsPolicyEngine {
return ret;
}
- public Set<String> getDatasetsForPrincipals(Set<String> users, Set<String> groups, Set<String> roles) {
- Set<String> ret = new HashSet<>();
+ public Set<Long> getDatasetsSharedWith(Set<String> users, Set<String> groups, Set<String> roles) {
+ Set<Long> ret = new HashSet<>();
for (GdsDatasetEvaluator dataset : datasets.values()) {
if (dataset.hasReference(users, groups, roles)) {
- ret.add(dataset.getName());
+ ret.add(dataset.getId());
}
}
return ret;
}
- public Set<String> getProjectsForPrincipals(Set<String> users, Set<String> groups, Set<String> roles) {
- Set<String> ret = new HashSet<>();
+ public Set<Long> getProjectsSharedWith(Set<String> users, Set<String> groups, Set<String> roles) {
+ Set<Long> ret = new HashSet<>();
for (GdsProjectEvaluator project : projects.values()) {
if (project.hasReference(users, groups, roles)) {
- ret.add(project.getName());
+ ret.add(project.getId());
}
}
@@ -159,21 +159,58 @@ public class GdsPolicyEngine {
return evaluator == null ? -1 : evaluator.getId();
}
+ public String getDatasetName(Long id) {
+ GdsDatasetEvaluator evaluator = datasets.get(id);
+
+ return evaluator == null ? null : evaluator.getName();
+ }
+
+ public String getProjectName(Long id) {
+ GdsProjectEvaluator evaluator = projects.get(id);
+
+ return evaluator == null ? null : evaluator.getName();
+ }
+
public Iterator<GdsSharedResourceEvaluator> getDatasetResources(long datasetId) {
- return new SharedResourceIter(getDataSharesForDataset(datasetId).listIterator());
+ Set<GdsDataShareEvaluator> dshEvaluators = new HashSet<>();
+
+ collectDataSharesForDataset(datasetId, dshEvaluators);
+
+ return new SharedResourceIter(dshEvaluators);
}
public Iterator<GdsSharedResourceEvaluator> getProjectResources(long projectId) {
- return new SharedResourceIter(getDataSharesForProject(projectId).listIterator());
+ Set<GdsDataShareEvaluator> dshEvaluators = new HashSet<>();
+
+ collectDataSharesForProject(projectId, dshEvaluators);
+
+ return new SharedResourceIter(dshEvaluators);
}
public Iterator<GdsSharedResourceEvaluator> getDataShareResources(long dataShareId) {
- GdsDataShareEvaluator evaluator = getDataShareEvaluator(dataShareId);
- List<GdsDataShareEvaluator> evaluators = evaluator == null ? Collections.emptyList() : Collections.singletonList(evaluator);
+ GdsDataShareEvaluator dshEvaluator = getDataShareEvaluator(dataShareId);
+ Set<GdsDataShareEvaluator> dshEvaluators = dshEvaluator == null ? Collections.emptySet() : Collections.singleton(dshEvaluator);
+
+ return new SharedResourceIter(dshEvaluators);
+ }
+
+ public Iterator<GdsSharedResourceEvaluator> getResources(List<Long> datasetIds, List<Long> dataShareIds) {
+ Set<GdsDataShareEvaluator> dshEvaluators = new HashSet<>();
+
+ collectDataShares(null, datasetIds, dataShareIds, dshEvaluators);
- return new SharedResourceIter(evaluators.listIterator());
+ return new SharedResourceIter(dshEvaluators);
}
+ public Iterator<GdsSharedResourceEvaluator> getResources(List<Long> projectIds, List<Long> datasetIds, List<Long> dataShareIds) {
+ Set<GdsDataShareEvaluator> dshEvaluators = new HashSet<>();
+
+ collectDataShares(projectIds, datasetIds, dataShareIds, dshEvaluators);
+
+ return new SharedResourceIter(dshEvaluators);
+ }
+
+
private void init(RangerServiceDefHelper serviceDefHelper, RangerPluginContext pluginContext) {
LOG.debug("==> RangerGdsPolicyEngine.init()");
@@ -423,63 +460,68 @@ public class GdsPolicyEngine {
return ret;
}
- private List<GdsDataShareEvaluator> getDataSharesForDataset(long datasetId) {
- Set<GdsDataShareEvaluator> evaluators = null;
-
+ private void collectDataSharesForDataset(long datasetId, Set<GdsDataShareEvaluator> evaluators) {
for (List<GdsDataShareEvaluator> dshEvaluators : zoneDataShares.values()) {
for (GdsDataShareEvaluator dshEvaluator : dshEvaluators) {
if (dshEvaluator.isInDataset(datasetId)) {
- if (evaluators == null) {
- evaluators = new HashSet<>();
- }
-
evaluators.add(dshEvaluator);
}
}
}
-
- List<GdsDataShareEvaluator> ret = (evaluators == null) ? Collections.emptyList() : new ArrayList<>(evaluators);
-
- if (ret.size() > 1) {
- ret.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR);
- }
-
- return ret;
}
- private List<GdsDataShareEvaluator> getDataSharesForProject(long projectId) {
- Set<GdsDataShareEvaluator> evaluators = null;
-
+ private void collectDataSharesForProject(long projectId, Set<GdsDataShareEvaluator> evaluators) {
for (List<GdsDataShareEvaluator> dshEvaluators : zoneDataShares.values()) {
for (GdsDataShareEvaluator dshEvaluator : dshEvaluators) {
if (dshEvaluator.isInProject(projectId)) {
- if (evaluators == null) {
- evaluators = new HashSet<>();
- }
-
evaluators.add(dshEvaluator);
}
}
}
+ }
- List<GdsDataShareEvaluator> ret = (evaluators == null) ? Collections.emptyList() : new ArrayList<>(evaluators);
+ private void collectDataShares(List<Long> projectIds, List<Long> datasetIds, List<Long> dataShareIds, Set<GdsDataShareEvaluator> evaluators) {
+ if (projectIds != null) {
+ for (Long projectId : projectIds) {
+ collectDataSharesForProject(projectId, evaluators);
+ }
+ }
- if (ret.size() > 1) {
- ret.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR);
+ if (datasetIds != null) {
+ for (Long datasetId : datasetIds) {
+ collectDataSharesForDataset(datasetId, evaluators);
+ }
+ }
+
+ if (dataShareIds != null) {
+ for (Long dataShareId : dataShareIds) {
+ GdsDataShareEvaluator evaluator = getDataShareEvaluator(dataShareId);
+
+ if (evaluator != null) {
+ evaluators.add(evaluator);
+ }
+ }
}
- return ret;
}
static class SharedResourceIter implements Iterator<GdsSharedResourceEvaluator> {
private final Iterator<GdsDataShareEvaluator> dataShareIter;
- private Iterator<GdsSharedResourceEvaluator> sharedResourceIter;
- private GdsSharedResourceEvaluator nextResource;
+ private Iterator<GdsSharedResourceEvaluator> sharedResourceIter = Collections.emptyIterator();
+ private GdsSharedResourceEvaluator nextResource = null;
+
+ SharedResourceIter(Set<GdsDataShareEvaluator> evaluators) {
+ if (evaluators == null || evaluators.isEmpty()) {
+ dataShareIter = Collections.emptyIterator();
+ } else if (evaluators.size() == 1) {
+ dataShareIter = evaluators.iterator();
+ } else {
+ List<GdsDataShareEvaluator> list = new ArrayList<>(evaluators);
- SharedResourceIter(Iterator<GdsDataShareEvaluator> dataShareIter) {
- this.dataShareIter = dataShareIter;
- this.sharedResourceIter = Collections.emptyIterator();
- this.nextResource = null;
+ list.sort(GdsDataShareEvaluator.EVAL_ORDER_COMPARATOR);
+
+ dataShareIter = list.iterator();
+ }
setNext();
}
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
index 548963838..719ec3005 100755
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
@@ -126,7 +126,8 @@ public class SearchFilter {
public static final String UPDATE_TIME_START = "updatedTimeStart"; // search
public static final String UPDATE_TIME_END = "updatedTimeEnd"; // search
public static final String IS_DISTINCT = "isDistinct"; // search, sort
- public static final String RETRIEVE_ALL_PAGES = "retrieveAllPages"; // search
+ public static final String RETRIEVE_ALL_PAGES = "retrieveAllPages"; // search
+ public static final String SHARED_WITH_ME = "sharedWithMe"; // search
private Map<String, String> params;
private int startIndex;
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java
index fb6d593e9..71880dabf 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/gds/TestGdsPolicyEngine.java
@@ -99,19 +99,19 @@ public class TestGdsPolicyEngine {
assertEquals(test.name, test.acls, acls);
}
- } else if (test.principals != null) {
- Set<String> users = test.principals.get("users");
- Set<String> groups = test.principals.get("groups");
- Set<String> roles = test.principals.get("roles");
+ } else if (test.sharedWith != null) {
+ Set<String> users = test.sharedWith.get("users");
+ Set<String> groups = test.sharedWith.get("groups");
+ Set<String> roles = test.sharedWith.get("roles");
if (test.datasets != null) {
- Set<String> datasets = policyEngine.getDatasetsForPrincipals(users, groups, roles);
+ Set<Long> datasets = policyEngine.getDatasetsSharedWith(users, groups, roles);
assertEquals(test.name, test.datasets, datasets);
}
if (test.projects != null) {
- Set<String> projects = policyEngine.getProjectsForPrincipals(users, groups, roles);
+ Set<Long> projects = policyEngine.getProjectsSharedWith(users, groups, roles);
assertEquals(test.name, test.projects, projects);
}
@@ -133,6 +133,12 @@ public class TestGdsPolicyEngine {
} else if (test.dataShareId != null) {
Iterator<GdsSharedResourceEvaluator> iter = policyEngine.getDataShareResources(test.dataShareId);
+ while (iter.hasNext()) {
+ resourceIds.add(iter.next().getId());
+ }
+ } else if (test.projectIds != null || test.datasetIds != null || test.dataShareIds != null) {
+ Iterator<GdsSharedResourceEvaluator> iter = policyEngine.getResources(test.projectIds, test.datasetIds, test.dataShareIds);
+
while (iter.hasNext()) {
resourceIds.add(iter.next().getId());
}
@@ -156,12 +162,15 @@ public class TestGdsPolicyEngine {
public RangerAccessRequest request;
public GdsAccessResult result;
public RangerResourceACLs acls;
- public Map<String, Set<String>> principals;
- public Set<String> datasets;
- public Set<String> projects;
+ public Map<String, Set<String>> sharedWith; // principals
+ public Set<Long> datasets;
+ public Set<Long> projects;
public Long datasetId;
public Long projectId;
public Long dataShareId;
+ public List<Long> datasetIds;
+ public List<Long> projectIds;
+ public List<Long> dataShareIds;
public Set<Long> resourceIds;
}
diff --git a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json
index bcdf5a342..71ecd225a 100644
--- a/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json
+++ b/agents-common/src/test/resources/policyengine/gds/test_gds_policy_engine_hive.json
@@ -370,53 +370,53 @@
{
"name": "Datasets for principals: users[ ds-user ]",
- "principals": { "users": [ "ds-user" ] },
- "datasets": [ "dataset-1", "dataset-2", "dataset-3", "dataset-4" ]
+ "sharedWith": { "users": [ "ds-user" ] },
+ "datasets": [ 1, 2, 3, 4 ]
},
{
"name": "Datasets for principals: users[ ds1-user ]",
- "principals": { "users": [ "ds1-user" ] },
- "datasets": [ "dataset-1" ]
+ "sharedWith": { "users": [ "ds1-user" ] },
+ "datasets": [ 1 ]
},
{
"name": "Datasets for principals: users[ ds2-user ]",
- "principals": { "users": [ "ds2-user" ] },
- "datasets": [ "dataset-2" ]
+ "sharedWith": { "users": [ "ds2-user" ] },
+ "datasets": [ 2 ]
},
{
"name": "Datasets for principals: users[ ds3-user ]",
- "principals": { "users": [ "ds3-user" ] },
- "datasets": [ "dataset-3" ]
+ "sharedWith": { "users": [ "ds3-user" ] },
+ "datasets": [ 3 ]
},
{
"name": "Datasets for principals: users[ ds4-user ]",
- "principals": { "users": [ "ds4-user" ] },
- "datasets": [ "dataset-4" ]
+ "sharedWith": { "users": [ "ds4-user" ] },
+ "datasets": [ 4 ]
},
{
"name": "Datasets for principals: users[ ds1-user, ds4-user ]",
- "principals": { "users": [ "ds1-user","ds4-user" ] },
- "datasets": [ "dataset-1", "dataset-4" ]
+ "sharedWith": { "users": [ "ds1-user","ds4-user" ] },
+ "datasets": [ 1, 4 ]
},
{
"name": "Projects for principals: users=[ proj-user ]",
"principals:": { "users": [ "proj-user" ] },
- "projects": [ "project-1", "project-2" ]
+ "projects": [ 1, 2 ]
},
{
"name": "Projects for principals: users[ proj1-user ]",
- "principals": { "users": [ "proj1-user" ] },
- "projects": [ "project-1" ]
+ "sharedWith": { "users": [ "proj1-user" ] },
+ "projects": [ 1 ]
},
{
"name": "Projects for principals: users[ proj2-user ]",
- "principals": { "users": [ "proj2-user" ] },
- "projects": [ "project-2" ]
+ "sharedWith": { "users": [ "proj2-user" ] },
+ "projects": [ 2 ]
},
{
"name": "Projects for principals: users[ proj1-user, proj2-user ]",
- "principals": { "users": [ "proj1-user", "proj2-user" ] },
- "projects": [ "project-1", "project-2" ]
+ "sharedWith": { "users": [ "proj1-user", "proj2-user" ] },
+ "projects": [ 1, 2 ]
},
{
@@ -490,6 +490,28 @@
"name": "Resources for DataShare: id=1234 (non_existent_data_share)",
"dataShareId": 1234,
"resourceIds": [ ]
+ },
+ {
+ "name": "Resources for projects(1, 2)",
+ "projectIds": [ 1, 2 ],
+ "resourceIds": [ 11, 12, 21, 22, 31, 41 ]
+ },
+ {
+ "name": "Resources for datasets(1, 2)",
+ "datasetIds": [ 1, 2 ],
+ "resourceIds": [ 11, 12, 21, 22, 31 ]
+ },
+ {
+ "name": "Resources for datasets(3, 4)",
+ "datasetIds": [ 3, 4 ],
+ "resourceIds": [ 41, 51 ]
+ },
+ {
+ "name": "Resources for projects(1, 2), datasets(1, 2, 3, 4), dataShares(1, 2, 3, 4, 5)",
+ "projectIds": [ 1, 2 ],
+ "datasetIds": [ 1, 2, 3, 4 ],
+ "dataShareIds": [ 1, 2, 3, 4, 5 ],
+ "resourceIds": [ 11, 12, 21, 22, 31, 41, 51 ]
}
]
}
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
index 292caf981..af70daa9d 100755
--- a/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsDBStore.java
@@ -37,6 +37,7 @@ import org.apache.ranger.plugin.store.PList;
import org.apache.ranger.plugin.store.ServiceStore;
import org.apache.ranger.plugin.util.*;
import org.apache.ranger.service.*;
+import org.apache.ranger.validation.RangerGdsValidationDBProvider;
import org.apache.ranger.validation.RangerGdsValidator;
import org.apache.ranger.view.RangerGdsVList.*;
import org.slf4j.Logger;
@@ -69,6 +70,9 @@ public class GdsDBStore extends AbstractGdsStore {
@Autowired
RangerGdsValidator validator;
+ @Autowired
+ RangerGdsValidationDBProvider validationDBProvider;
+
@Autowired
RangerDaoManager daoMgr;
@@ -108,6 +112,9 @@ public class GdsDBStore extends AbstractGdsStore {
@Autowired
ServiceGdsInfoCache serviceGdsInfoCache;
+ @Autowired
+ GdsPolicyAdminCache gdsPolicyAdminCache;
+
@PostConstruct
public void initStore() {
if (LOG.isDebugEnabled()) {
@@ -270,10 +277,9 @@ public class GdsDBStore extends AbstractGdsStore {
LOG.debug("==> searchDatasets({})", filter);
PList<RangerDataset> ret = getUnscrubbedDatasets(filter);
- List<RangerDataset> datasets = ret.getList();
GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
- for (RangerDataset dataset : datasets) {
+ for (RangerDataset dataset : ret.getList()) {
if (gdsPermission.equals(GdsPermission.LIST)) {
scrubDatasetForListing(dataset);
}
@@ -562,23 +568,15 @@ public class GdsDBStore extends AbstractGdsStore {
public PList<RangerProject> searchProjects(SearchFilter filter) {
LOG.debug("==> searchProjects({})", filter);
- int maxRows = filter.getMaxRows();
- int startIndex = filter.getStartIndex();
-
- GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
- RangerProjectList result = projectService.searchProjects(filter);
- List<RangerProject> projects = new ArrayList<>();
+ GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
+ PList<RangerProject> ret = getUnscrubbedProjects(filter);
- for (RangerProject project : result.getList()) {
+ for (RangerProject project : ret.getList()) {
if (gdsPermission.equals(GdsPermission.LIST)) {
scrubProjectForListing(project);
}
-
- projects.add(project);
}
- PList<RangerProject> ret = getPList(projects, startIndex, maxRows, result.getSortBy(), result.getSortType());
-
LOG.debug("<== searchProjects({}): ret={}", filter, ret);
return ret;
@@ -1254,10 +1252,6 @@ public class GdsDBStore extends AbstractGdsStore {
for (RangerDataset dataset : datasets) {
GdsPermission permissionForCaller = validator.getGdsPermissionForUser(dataset.getAcl(), currentUser);
- if (permissionForCaller.equals(GdsPermission.NONE)) {
- continue;
- }
-
DatasetSummary datasetSummary = new DatasetSummary();
datasetSummary.setId(dataset.getId());
@@ -1374,30 +1368,76 @@ public class GdsDBStore extends AbstractGdsStore {
return ret;
}
- private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
- int maxRows = filter.getMaxRows();
- int startIndex = filter.getStartIndex();
+ private PList<RangerProject> getUnscrubbedProjects(SearchFilter filter) {
+ filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");
+
+ GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
+ RangerProjectList result = projectService.searchProjects(filter);
+ List<RangerProject> projects = new ArrayList<>();
+ boolean isSharedWithMe = Boolean.parseBoolean(filter.getParam(SearchFilter.SHARED_WITH_ME));
+ String userName = bizUtil.getCurrentUserLoginId();
+ Collection<String> groups = null;
+ Collection<String> roles = null;
+
+ if (isSharedWithMe) {
+ groups = validationDBProvider.getGroupsForUser(userName);
+ roles = validationDBProvider.getRolesForUser(userName);
+ }
+ for (RangerProject project : result.getList()) {
+ if (project == null) {
+ continue;
+ }
+
+ if (isSharedWithMe) {
+ if (gdsPolicyAdminCache.isProjectSharedWith(project.getId(), userName, groups, roles)) {
+ projects.add(project);
+ }
+ } else if (validator.hasPermission(project.getAcl(), gdsPermission)) {
+ projects.add(project);
+ }
+ }
+
+ return getPList(projects, filter.getStartIndex(), filter.getMaxRows(), result.getSortBy(), result.getSortType());
+ }
+
+ private PList<RangerDataset> getUnscrubbedDatasets(SearchFilter filter) {
filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");
- GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
- RangerDatasetList result = datasetService.searchDatasets(filter);
- List<RangerDataset> datasets = new ArrayList<>();
+ GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
+ RangerDatasetList result = datasetService.searchDatasets(filter);
+ List<RangerDataset> datasets = new ArrayList<>();
+ boolean isSharedWithMe = Boolean.parseBoolean(filter.getParam(SearchFilter.SHARED_WITH_ME));
+ String userName = bizUtil.getCurrentUserLoginId();
+ Collection<String> groups = null;
+ Collection<String> roles = null;
+ if (isSharedWithMe) {
+ groups = validationDBProvider.getGroupsForUser(userName);
+ roles = validationDBProvider.getRolesForUser(userName);
+ }
for (RangerDataset dataset : result.getList()) {
- if (dataset != null && validator.hasPermission(dataset.getAcl(), gdsPermission)) {
+ if (dataset == null) {
+ continue;
+ }
+
+ if (isSharedWithMe) {
+ if (gdsPolicyAdminCache.isDatasetSharedWith(dataset.getId(), userName, groups, roles)) {
+ datasets.add(dataset);
+ }
+ } else if (validator.hasPermission(dataset.getAcl(), gdsPermission)) {
datasets.add(dataset);
}
}
+ int maxRows = filter.getMaxRows();
+ int startIndex = filter.getStartIndex();
+
return getPList(datasets, startIndex, maxRows, result.getSortBy(), result.getSortType());
}
private PList<RangerDataShare> getUnscrubbedDataShares(SearchFilter filter) {
- int maxRows = filter.getMaxRows();
- int startIndex = filter.getStartIndex();
-
filter.setParam(SearchFilter.RETRIEVE_ALL_PAGES, "true");
String datasetId = filter.getParam(SearchFilter.DATASET_ID);
@@ -1414,20 +1454,23 @@ public class GdsDBStore extends AbstractGdsStore {
dataSharesToExclude = Collections.emptyList();
}
- GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
- RangerDataShareList result = dataShareService.searchDataShares(filter);
- List<RangerDataShare> dataShares = new ArrayList<>();
-
+ GdsPermission gdsPermission = getGdsPermissionFromFilter(filter);
+ RangerDataShareList result = dataShareService.searchDataShares(filter);
+ List<RangerDataShare> dataShares = new ArrayList<>();
for (RangerDataShare dataShare : result.getList()) {
- if (dataShare != null && validator.hasPermission(dataShare.getAcl(), gdsPermission)) {
+ if (dataShare == null) {
+ continue;
+ }
+
+ if (validator.hasPermission(dataShare.getAcl(), gdsPermission)) {
if (!dataSharesToExclude.contains(dataShare.getId())) {
dataShares.add(dataShare);
}
}
}
- return getPList(dataShares, startIndex, maxRows, result.getSortBy(), result.getSortType());
+ return getPList(dataShares, filter.getStartIndex(), filter.getMaxRows(), result.getSortBy(), result.getSortType());
}
private <T> PList<T> getPList(List<T> list, int startIndex, int maxEntries, String sortBy, String sortType) {
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/GdsPolicyAdminCache.java b/security-admin/src/main/java/org/apache/ranger/biz/GdsPolicyAdminCache.java
new file mode 100644
index 000000000..97d4b2579
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/biz/GdsPolicyAdminCache.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.biz;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
+import org.apache.ranger.plugin.policyengine.gds.GdsPolicyEngine;
+import org.apache.ranger.plugin.util.ServicePolicies;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+
+
+@Component
+public class GdsPolicyAdminCache {
+ private static final Logger LOG = LoggerFactory.getLogger(GdsPolicyAdminCache.class);
+
+ @Autowired
+ ServiceDBStore svcStore;
+
+ private volatile GdsPolicies policies = null;
+
+ public GdsPolicyAdminCache() {
+ }
+
+ public boolean isDatasetSharedWith(Long datasetId, String user, Collection<String> groups, Collection<String> roles) {
+ boolean ret = false;
+ List<RangerPolicy> policies = getDatasetPolicies(datasetId);
+
+ for (RangerPolicy policy : policies) {
+ ret = hasReference(policy, user, groups, roles);
+
+ if (ret) {
+ break;
+ }
+ }
+
+ LOG.debug("isDatasetSharedWith(datasetId={}, user={}, groups={}, roles={}): policies={}, ret={}", datasetId, user, groups, roles, policies, ret);
+
+ return ret;
+ }
+
+ public boolean isProjectSharedWith(Long projectId, String user, Collection<String> groups, Collection<String> roles) {
+ boolean ret = false;
+ List<RangerPolicy> policies = getProjectPolicies(projectId);
+
+ for (RangerPolicy policy : policies) {
+ ret = hasReference(policy, user, groups, roles);
+
+ if (ret) {
+ break;
+ }
+ }
+
+ LOG.debug("isProjectSharedWith(datasetId={}, user={}, groups={}, roles={}): policies={}, ret={}", projectId, user, groups, roles, policies, ret);
+
+ return ret;
+ }
+
+ private boolean hasReference(RangerPolicy policy, String user, Collection<String> groups, Collection<String> roles) {
+ boolean ret = false;
+
+ if (policy.getPolicyItems() != null) {
+ for (RangerPolicyItem policyItem : policy.getPolicyItems()) {
+ if (policyItem == null) {
+ continue;
+ }
+
+ ret = policyItem.getUsers() != null && policyItem.getUsers().contains(user);
+
+ if (!ret && groups != null && policyItem.getGroups() != null) {
+ ret = CollectionUtils.containsAny(groups, policyItem.getGroups());
+ }
+
+ if (!ret && roles != null && policyItem.getRoles() != null) {
+ ret = CollectionUtils.containsAny(roles, policyItem.getRoles());
+ }
+
+ if (ret) {
+ break;
+ }
+ }
+ }
+
+ LOG.debug("hasReference(policy={}, user={}, groups={}, roles={}):, ret={}", policy, user, groups, roles, ret);
+
+ return ret;
+ }
+
+ private List<RangerPolicy> getDatasetPolicies(Long datasetId) {
+ GdsPolicies policies = getLatestGdsPolicies();
+ List<RangerPolicy> ret = policies != null && policies.datasetPolicies != null ? policies.datasetPolicies.get(datasetId) : Collections.emptyList();
+
+ return ret != null ? ret : Collections.emptyList();
+ }
+
+ private List<RangerPolicy> getProjectPolicies(Long projectId) {
+ GdsPolicies policies = getLatestGdsPolicies();
+ List<RangerPolicy> ret = policies != null && policies.projectPolicies != null ? policies.projectPolicies.get(projectId) : Collections.emptyList();
+
+ return ret != null ? ret : Collections.emptyList();
+ }
+
+ private synchronized GdsPolicies getLatestGdsPolicies() {
+ try {
+ Long currentVersion = policies != null ? policies.policiesVersion : null;
+ ServicePolicies latestPolicies = svcStore.getServicePoliciesIfUpdated(GdsPolicyEngine.GDS_SERVICE_NAME, currentVersion != null ? currentVersion : -1L, true);
+
+ if (latestPolicies != null && latestPolicies.getPolicies() != null) {
+ Long latestVersion = latestPolicies.getPolicyVersion();
+ Map<Long, List<RangerPolicy>> datasetPolicies = new HashMap<>();
+ Map<Long, List<RangerPolicy>> projectPolicies = new HashMap<>();
+
+ LOG.info("refreshing GDS policies from version {} to version {}", currentVersion, latestVersion);
+
+ for (RangerPolicy policy : latestPolicies.getPolicies()) {
+ if (policy != null && (policy.getIsEnabled() == null || Boolean.TRUE.equals(policy.getIsEnabled())) && policy.getResources() != null) {
+ if (policy.getResources().containsKey(GdsPolicyEngine.RESOURCE_NAME_DATASET_ID)) {
+ Long datasetId = getId(policy.getResources().get(GdsPolicyEngine.RESOURCE_NAME_DATASET_ID));
+
+ if (datasetId == null) {
+ LOG.warn("getLatestGdsPolicies(): ignoring dataset policy id={} - failed to get dataset-id from resource={}", policy.getId(), policy.getResources());
+ continue;
+ }
+
+ datasetPolicies.computeIfAbsent(datasetId, k -> new ArrayList<>()).add(policy);
+ } else if (policy.getResources().containsKey(GdsPolicyEngine.RESOURCE_NAME_PROJECT_ID)) {
+ Long projectId = getId(policy.getResources().get(GdsPolicyEngine.RESOURCE_NAME_PROJECT_ID));
+
+ if (projectId == null) {
+ LOG.warn("getLatestGdsPolicies(): ignoring project policy id={} - failed to get project-id from resource={}", policy.getId(), policy.getResources());
+ continue;
+ }
+
+ projectPolicies.computeIfAbsent(projectId, k -> new ArrayList<>()).add(policy);
+ } else {
+ LOG.warn("getLatestGdsPolicies(): ignoring policy id={} - does not have dataset-id or project-id in resource={}", policy.getId(), policy.getResources());
+ }
+ }
+ }
+
+ this.policies = new GdsPolicies(latestVersion, datasetPolicies, projectPolicies);
+ } else {
+ LOG.debug("getLatestGdsPolicies(): no changes since currentVersion={}", currentVersion);
+ }
+ } catch (Exception excp) {
+ LOG.error("getLatestGdsPolicies(): failed to get policies", excp);
+ }
+
+ return this.policies;
+ }
+
+ private Long getId(RangerPolicy.RangerPolicyResource policyResource) {
+ Long ret = null;
+
+ if (policyResource != null && policyResource.getValues() != null) {
+ for (String value : policyResource.getValues()) {
+ try {
+ ret = Long.parseLong(value);
+ } catch (NumberFormatException excp) {
+ // ignore
+ }
+
+ if (ret != null) {
+ break;
+ }
+ }
+
+ if (ret == null) {
+ LOG.warn("getId(): failed to parse dataset/project id from resource-value={}", policyResource.getValues());
+ }
+ }
+
+ return ret;
+ }
+
+ private static class GdsPolicies {
+ private final Long policiesVersion;
+ private final Map<Long, List<RangerPolicy>> datasetPolicies;
+ private final Map<Long, List<RangerPolicy>> projectPolicies;
+
+ public GdsPolicies(Long policiesVersion, Map<Long, List<RangerPolicy>> datasetPolicies, Map<Long, List<RangerPolicy>> projectPolicies) {
+ this.policiesVersion = policiesVersion;
+ this.datasetPolicies = datasetPolicies;
+ this.projectPolicies = projectPolicies;
+ }
+ }
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
index 8c9a6c13d..87a278674 100755
--- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
@@ -126,6 +126,7 @@ public class RangerSearchUtil extends SearchUtil {
ret.setParam(SearchFilter.UPDATE_TIME_START, request.getParameter(SearchFilter.UPDATE_TIME_START));
ret.setParam(SearchFilter.UPDATE_TIME_END, request.getParameter(SearchFilter.UPDATE_TIME_END));
ret.setParam(SearchFilter.RESOURCE_CONTAINS, request.getParameter(SearchFilter.RESOURCE_CONTAINS));
+ ret.setParam(SearchFilter.SHARED_WITH_ME, request.getParameter(SearchFilter.SHARED_WITH_ME));
extractCommonCriteriasForFilter(request, ret, sortFields);
diff --git a/security-admin/src/main/java/org/apache/ranger/common/ServiceGdsInfoCache.java b/security-admin/src/main/java/org/apache/ranger/common/ServiceGdsInfoCache.java
index b27a32265..a3b6ab3ce 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/ServiceGdsInfoCache.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/ServiceGdsInfoCache.java
@@ -20,17 +20,15 @@
package org.apache.ranger.common;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXServiceVersionInfo;
import org.apache.ranger.plugin.model.RangerGds;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.store.ServiceStore;
-import org.apache.ranger.plugin.util.RangerCache;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServiceGdsInfo;
import org.apache.ranger.service.*;
-import org.apache.ranger.util.RangerCacheDBValueLoader;
+import org.apache.ranger.util.RangerAdminCache;
import org.apache.ranger.view.RangerGdsVList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,17 +45,9 @@ import java.util.List;
import static org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_GDS_NAME;
@Component
-public class ServiceGdsInfoCache extends RangerCache<String, ServiceGdsInfo> {
+public class ServiceGdsInfoCache extends RangerAdminCache<String, ServiceGdsInfo> {
private static final Logger LOG = LoggerFactory.getLogger(ServiceGdsInfoCache.class);
- public static final String PROP_LOADER_THREAD_POOL_SIZE = "ranger.admin.cache.gds-info.loader.threadpool.size";
- public static final String PROP_VALUE_INIT_TIMEOUT_MS = "ranger.admin.cache.gds-info.value.init.timeout.ms";
- public static final String PROP_VALUE_REFRESH_TIMEOUT_MS = "ranger.admin.cache.gds-info.value.refresh.timeout.ms";
-
- private static final int DEFAULT_LOADER_THREAD_POOL_SIZE = 1;
- private static final int DEFAULT_VALUE_INIT_LOAD_TIMEOUT_MS = -1;
- private static final int DEFAULT_VALUE_REFRESH_LOAD_TIMEOUT_MS = 10 * 1000; // 10 seconds
-
@Autowired
RangerDaoManager daoMgr;
@@ -87,13 +77,7 @@ public class ServiceGdsInfoCache extends RangerCache<String, ServiceGdsInfo> {
PlatformTransactionManager txManager;
public ServiceGdsInfoCache() {
- super("ServiceGdsInfoCache",
- null, // loader will be set in init(), so that txManager is initialized
- getLoaderThreadPoolSize(),
- RefreshMode.ON_ACCESS,
- 0, // every access should look to refresh
- getValueInitLoadTimeout(),
- getValueRefreshLoadTimeout());
+ super("gds-info", null);
}
@PostConstruct
@@ -102,19 +86,7 @@ public class ServiceGdsInfoCache extends RangerCache<String, ServiceGdsInfo> {
}
- private static int getLoaderThreadPoolSize() {
- return RangerAdminConfig.getInstance().getInt(PROP_LOADER_THREAD_POOL_SIZE, DEFAULT_LOADER_THREAD_POOL_SIZE);
- }
-
- private static long getValueInitLoadTimeout() {
- return RangerAdminConfig.getInstance().getLong(PROP_VALUE_INIT_TIMEOUT_MS, DEFAULT_VALUE_INIT_LOAD_TIMEOUT_MS);
- }
-
- private static long getValueRefreshLoadTimeout() {
- return RangerAdminConfig.getInstance().getLong(PROP_VALUE_REFRESH_TIMEOUT_MS, DEFAULT_VALUE_REFRESH_LOAD_TIMEOUT_MS);
- }
-
- private class ServiceGdsInfoLoader extends RangerCacheDBValueLoader<String, ServiceGdsInfo> {
+ private class ServiceGdsInfoLoader extends RangerDBValueLoader<String, ServiceGdsInfo> {
public ServiceGdsInfoLoader(PlatformTransactionManager txManager) {
super(txManager);
}
diff --git a/security-admin/src/main/java/org/apache/ranger/util/RangerAdminCache.java b/security-admin/src/main/java/org/apache/ranger/util/RangerAdminCache.java
new file mode 100644
index 000000000..569c1133a
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/util/RangerAdminCache.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.util;
+
+import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
+import org.apache.ranger.plugin.util.RangerCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
+
+public class RangerAdminCache<K, V> extends RangerCache<K, V> {
+ private static final Logger LOG = LoggerFactory.getLogger(RangerDBValueLoader.class);
+
+ public static final int DEFAULT_ADMIN_CACHE_LOADER_THREADS_COUNT = 1;
+ public static final RefreshMode DEFAULT_ADMIN_CACHE_REFRESH_MODE = RefreshMode.ON_ACCESS;
+ public static final long DEFAULT_ADMIN_CACHE_VALUE_VALIDITY_PERIOD_MS = 0; // every access should look to refresh
+ public static final long DEFAULT_ADMIN_CACHE_VALUE_INIT_TIMEOUT_MS = -1L; // infinite timeout
+ public static final long DEFAULT_ADMIN_CACHE_VALUE_REFRESH_TIMEOUT_MS = 10 * 1000L; // 10 seconds
+
+ private static final String PROP_PREFIX = "ranger.admin.cache.";
+ private static final String PROP_LOADER_THREAD_POOL_SIZE = ".loader.threadpool.size";
+ private static final String PROP_VALUE_INIT_TIMEOUT_MS = ".value.init.timeout.ms";
+ private static final String PROP_VALUE_REFRESH_TIMEOUT_MS = ".value.refresh.timeout.ms";
+
+ protected RangerAdminCache(String name, RangerDBValueLoader<K, V> loader) {
+ this(name, loader, getLoaderThreadPoolSize(name), DEFAULT_ADMIN_CACHE_REFRESH_MODE, DEFAULT_ADMIN_CACHE_VALUE_VALIDITY_PERIOD_MS, getValueInitLoadTimeout(name), getValueRefreshLoadTimeout(name));
+ }
+
+ protected RangerAdminCache(String name, RangerDBValueLoader<K, V> loader, int loaderThreadsCount, RefreshMode refreshMode, long valueValidityPeriodMs, long valueInitLoadTimeoutMs, long valueRefreshLoadTimeoutMs) {
+ super(name, loader, loaderThreadsCount, refreshMode, valueValidityPeriodMs, valueInitLoadTimeoutMs, valueRefreshLoadTimeoutMs);
+ }
+
+ private static int getLoaderThreadPoolSize(String cacheName) {
+ return RangerAdminConfig.getInstance().getInt(PROP_PREFIX + cacheName + PROP_LOADER_THREAD_POOL_SIZE, DEFAULT_ADMIN_CACHE_LOADER_THREADS_COUNT);
+ }
+
+ private static long getValueInitLoadTimeout(String cacheName) {
+ return RangerAdminConfig.getInstance().getLong(PROP_PREFIX + cacheName + PROP_VALUE_INIT_TIMEOUT_MS, DEFAULT_ADMIN_CACHE_VALUE_INIT_TIMEOUT_MS);
+ }
+
+ private static long getValueRefreshLoadTimeout(String cacheName) {
+ return RangerAdminConfig.getInstance().getLong(PROP_PREFIX + cacheName + PROP_VALUE_REFRESH_TIMEOUT_MS, DEFAULT_ADMIN_CACHE_VALUE_REFRESH_TIMEOUT_MS);
+ }
+
+ public abstract static class RangerDBValueLoader<K, V> extends ValueLoader<K, V> {
+ private final TransactionTemplate txTemplate;
+
+ public RangerDBValueLoader(PlatformTransactionManager txManager) {
+ this.txTemplate = new TransactionTemplate(txManager);
+
+ txTemplate.setReadOnly(true);
+ }
+
+ @Override
+ final public RefreshableValue<V> load(K key, RefreshableValue<V> currentValue) throws Exception {
+ Exception[] ex = new Exception[1];
+
+ RefreshableValue<V> ret = txTemplate.execute(status -> {
+ try {
+ return dbLoad(key, currentValue);
+ } catch (Exception excp) {
+ LOG.error("RangerDBLoaderCache.load(): failed to load for key={}", key, excp);
+
+ ex[0] = excp;
+ }
+
+ return null;
+ });
+
+ if (ex[0] != null) {
+ throw ex[0];
+ }
+
+ return ret;
+ }
+
+ protected abstract RefreshableValue<V> dbLoad(K key, RefreshableValue<V> currentValue) throws Exception;
+ }
+}
diff --git a/security-admin/src/main/java/org/apache/ranger/util/RangerCacheDBValueLoader.java b/security-admin/src/main/java/org/apache/ranger/util/RangerCacheDBValueLoader.java
deleted file mode 100644
index 7b5da29c3..000000000
--- a/security-admin/src/main/java/org/apache/ranger/util/RangerCacheDBValueLoader.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ranger.util;
-
-import org.apache.ranger.plugin.util.RangerCache;
-import org.apache.ranger.plugin.util.RangerCache.RefreshableValue;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.transaction.PlatformTransactionManager;
-import org.springframework.transaction.support.TransactionTemplate;
-
-public abstract class RangerCacheDBValueLoader<K, V> extends RangerCache.ValueLoader<K, V> {
- private static final Logger LOG = LoggerFactory.getLogger(RangerCacheDBValueLoader.class);
-
- private final TransactionTemplate txTemplate;
-
- public RangerCacheDBValueLoader(PlatformTransactionManager txManager) {
- this.txTemplate = new TransactionTemplate(txManager);
-
- txTemplate.setReadOnly(true);
- }
-
- @Override
- public RefreshableValue<V> load(K key, RefreshableValue<V> currentValue) throws Exception {
- Exception[] ex = new Exception[1];
-
- RefreshableValue<V> ret = txTemplate.execute(status -> {
- try {
- return dbLoad(key, currentValue);
- } catch (Exception excp) {
- LOG.error("RangerDBLoaderCache.load(): failed to load for key={}", key, excp);
-
- ex[0] = excp;
- }
-
- return null;
- });
-
- if (ex[0] != null) {
- throw ex[0];
- }
-
- return ret;
- }
-
- protected abstract RefreshableValue<V> dbLoad(K key, RefreshableValue<V> currentValue) throws Exception;
-}