You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/05/20 05:17:22 UTC
[2/2] incubator-ranger git commit: RANGER-274: Added service-def for
Tags; added TagStore/TagFileStore, TagREST to add/retrieve tags for resources
RANGER-274: Added service-def for Tags; added TagStore/TagFileStore, TagREST to add/retrieve tags for resources
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/9578f94c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/9578f94c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/9578f94c
Branch: refs/heads/tag-policy
Commit: 9578f94cdf0dd0e6ea31bb4c524edfda82576ef4
Parents: b5a23b2
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon May 18 14:34:21 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue May 19 20:14:43 2015 -0700
----------------------------------------------------------------------
.../ranger/plugin/model/RangerResource.java | 143 +++++
.../ranger/plugin/model/RangerServiceDef.java | 7 +
.../ranger/plugin/model/RangerTagDef.java | 133 +++++
.../plugin/store/AbstractServiceStore.java | 372 ++++++++++++
.../ranger/plugin/store/AbstractTagStore.java | 88 +++
.../plugin/store/EmbeddedServiceDefsUtil.java | 11 +-
.../org/apache/ranger/plugin/store/PList.java | 189 ++++++
.../plugin/store/RangerPolicyPaginatedList.java | 62 ++
.../store/RangerServiceDefPaginatedList.java | 61 ++
.../store/RangerServicePaginatedList.java | 62 ++
.../ranger/plugin/store/ServiceStore.java | 22 +-
.../ranger/plugin/store/TagPredicateUtil.java | 212 +++++++
.../apache/ranger/plugin/store/TagStore.java | 59 ++
.../ranger/plugin/store/file/BaseFileStore.java | 395 -------------
.../ranger/plugin/store/file/FileStoreUtil.java | 317 +++++++++++
.../plugin/store/file/ServiceFileStore.java | 59 +-
.../ranger/plugin/store/file/TagFileStore.java | 569 +++++++++++++++++++
.../plugin/store/rest/ServiceRESTStore.java | 3 +-
.../apache/ranger/plugin/util/SearchFilter.java | 9 +
.../service-defs/ranger-servicedef-_tag_.json | 73 +++
.../org/apache/ranger/biz/ServiceDBStore.java | 68 ++-
.../org/apache/ranger/common/view/VList.java | 7 +-
.../org/apache/ranger/rest/ServiceREST.java | 73 ++-
.../java/org/apache/ranger/rest/TagREST.java | 409 +++++++++++++
.../apache/ranger/rest/TagRESTConstants.java | 39 ++
.../ranger/service/RangerServiceDefService.java | 33 +-
.../apache/ranger/biz/TestServiceDBStore.java | 18 +-
.../org/apache/ranger/rest/TestServiceREST.java | 12 +-
28 files changed, 3029 insertions(+), 476 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java
new file mode 100644
index 0000000..23bb098
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerResource.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.model;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This class represents a RangerResource including the service-type (such as hdfs, hive, etc.) in which it is supported.
+ * This implies that there is one-to-one mapping between service-type and the resource-type which is a valid assumption.
+ * Service-type must be one of service-types supported by Ranger.
+ *
+ * This class also contains a list of (tag-name, JSON-string-representing-tagattribute-tagattributevalue-pairs)
+ *
+ */
+
+@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
+@JsonIgnoreProperties(ignoreUnknown=true)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+
+public class RangerResource extends RangerBaseModelObject {
+ private static final long serialVersionUID = 1L;
+
+ private String serviceType = null; // one of any supported by any component
+ private Map<String, RangerPolicy.RangerPolicyResource> resourceSpec = null; //
+ private String tagServiceName = null;
+ private List<RangerResourceTag> tagsAndValues = null;
+
+ public RangerResource(String serviceType, Map<String, RangerPolicy.RangerPolicyResource> resourceSpecs, String tagServiceName, List<RangerResourceTag> tagsAndValues) {
+ super();
+ setServiceType(serviceType);
+ setResourceSpecs(resourceSpecs);
+ setTagServiceName(tagServiceName);
+ setTagsAndValues(tagsAndValues);
+ }
+
+ public RangerResource() {
+ this(null, null, null, null);
+ }
+
+ public String getServiceType() {
+ return serviceType;
+ }
+
+ public Map<String, RangerPolicy.RangerPolicyResource> getResourceSpecs() {
+ return resourceSpec;
+ }
+
+ public String getTagServiceName() {
+ return tagServiceName;
+ }
+
+ public List<RangerResourceTag> getTagsAndValues() {
+ return tagsAndValues;
+ }
+
+ // And corresponding set methods
+ public void setServiceType(String serviceType) {
+ this.serviceType = serviceType == null ? new String() : serviceType;
+ }
+
+ public void setResourceSpecs(Map<String, RangerPolicy.RangerPolicyResource> fullName) {
+ this.resourceSpec = resourceSpec == null ? new HashMap<String, RangerPolicy.RangerPolicyResource>() : resourceSpec;
+ }
+
+ public void setTagServiceName(String tagServiceName) {
+ this.tagServiceName = tagServiceName == null ? new String() : tagServiceName;
+ }
+
+ public void setTagsAndValues(List<RangerResourceTag> tagsAndValues) {
+ this.tagsAndValues = tagsAndValues == null ? new ArrayList<RangerResourceTag>() : tagsAndValues;
+ }
+
+ /**
+ * Represents a tag and its attribute-values for a resource.
+ */
+
+ @JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY)
+ @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
+ @JsonIgnoreProperties(ignoreUnknown=true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.FIELD)
+
+ public static class RangerResourceTag implements java.io.Serializable {
+
+ private String name = null;
+ private Map<String, Object> attributeValues = null; // Will be JSON string with (name, value) pairs of tag attributes in database
+
+ public RangerResourceTag(String name, Map<String, Object> attributeValues) {
+ super();
+ setName(name);
+ setAttributeValues(attributeValues);
+ }
+
+ public RangerResourceTag() {
+ this(null, null);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Map<String, Object> getAttributeValues() {
+ return attributeValues;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public void setAttributeValues(Map<String, Object> attributeValues) {
+ this.attributeValues = attributeValues;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java
index 302da2f..49ee3a1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerServiceDef.java
@@ -1672,6 +1672,13 @@ public class RangerServiceDef extends RangerBaseModelObject implements java.io.S
setImpliedGrants(impliedGrants);
}
+ public RangerAccessTypeDef(RangerAccessTypeDef other) {
+ this.setName(other.getName());
+ this.setLabel(other.getLabel());
+ this.setRbKeyLabel(other.getRbKeyLabel());
+ this.setImpliedGrants(other.getImpliedGrants());
+ }
+
/**
* @return the itemId
*/
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
new file mode 100644
index 0000000..0f58c96
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagDef.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.model;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.List;
+
+@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY)
+@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
+@JsonIgnoreProperties(ignoreUnknown=true)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+
+/**
+ * Represents a TAG definition known to Ranger. In general, this will be provided
+ * by some external system identified by 'source'.
+ *
+ */
+
+public class RangerTagDef extends RangerBaseModelObject {
+ private static final long serialVersionUID = 1L;
+
+ private String name = null;
+ private String source = null;
+
+ private List<RangerTagAttributeDef> attributeDefs;
+
+ public RangerTagDef() {
+ this(null, "Internal");
+ }
+
+ public RangerTagDef(String name) {
+ this(name, "Internal");
+ }
+
+ public RangerTagDef(String name, String source) {
+ super();
+ setName(name);
+ setSource(source);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+
+ this.name = name == null ? new String() : name;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source == null ? new String() : source;
+ }
+
+ public List<RangerTagAttributeDef> getAttributeDefs() {
+ return attributeDefs;
+ }
+
+ public void setAttributeDefs(List<RangerTagAttributeDef> attributeDefs) {
+ this.attributeDefs = attributeDefs == null ? new ArrayList<RangerTagAttributeDef>() : attributeDefs;
+ }
+
+ /**
+ * Represents one attribute for a TAG. TAG-Attribute consists of a name and type.
+ * name provides a handle for possible specification of additional information
+ * associated with the TAG.
+ * Interpretation of type is up to the policy-engine.
+ */
+
+ @JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY)
+ @JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL )
+ @JsonIgnoreProperties(ignoreUnknown=true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.FIELD)
+
+ public static class RangerTagAttributeDef implements java.io.Serializable {
+
+ private String name = null;
+ private String type = null;
+
+ public RangerTagAttributeDef() {
+ this(null, null);
+ }
+
+ public RangerTagAttributeDef(String name, String type) {
+ setName(name);
+ setType(type);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setName(String name) {
+ this.name = name == null ? new String() : name;
+ }
+ public void setType(String type) {
+ this.type = type == null ? new String() : type;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
new file mode 100644
index 0000000..e30535a
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.util.SearchFilter;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+public abstract class AbstractServiceStore implements ServiceStore {
+ private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class);
+
+
+ private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000;
+
+ @Override
+ public void updateTagServiceDefForAccessTypes() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceDefDBStore.updateTagServiceDefForAccessTypes()");
+ }
+ List<RangerServiceDef> allServiceDefs = getServiceDefs(new SearchFilter());
+ for (RangerServiceDef serviceDef : allServiceDefs) {
+ if (StringUtils.isEmpty(serviceDef.getName()) || serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ continue;
+ }
+ updateTagServiceDefForUpdatingAccessTypes(serviceDef);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDefDBStore.updateTagServiceDefForAccessTypes()");
+ }
+ return;
+ }
+
+ @Override
+ public void deleteServiceDef(Long id, Boolean forceDelete) throws Exception {
+ deleteServiceDef(id); // Ignore forceDelete flag
+ }
+
+ @Override
+ public RangerServiceDefPaginatedList getPaginatedServiceDefs(SearchFilter filter) throws Exception {
+ List<RangerServiceDef> resultList = getServiceDefs(filter);
+
+ RangerServiceDefPaginatedList ret = new RangerServiceDefPaginatedList();
+
+ ret.setResultSize(resultList.size());
+ ret.setPageSize(resultList.size());
+ ret.setSortBy(filter.getSortBy());
+ ret.setSortType(filter.getSortType());
+ ret.setStartIndex(0);
+ ret.setTotalCount(resultList.size());
+
+ ret.setServiceDefs(resultList);
+
+ return ret;
+ }
+
+ @Override
+ public RangerServicePaginatedList getPaginatedServices(SearchFilter filter) throws Exception {
+ List<RangerService> resultList = getServices(filter);
+
+ RangerServicePaginatedList ret = new RangerServicePaginatedList();
+
+ ret.setResultSize(resultList.size());
+ ret.setPageSize(resultList.size());
+ ret.setSortBy(filter.getSortBy());
+ ret.setSortType(filter.getSortType());
+ ret.setStartIndex(0);
+ ret.setTotalCount(resultList.size());
+
+ ret.setServices(resultList);
+
+ return ret;
+ }
+
+ @Override
+ public RangerPolicyPaginatedList getPaginatedPolicies(SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getPolicies(filter);
+
+ RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList();
+
+ ret.setResultSize(resultList.size());
+ ret.setPageSize(resultList.size());
+ ret.setSortBy(filter.getSortBy());
+ ret.setSortType(filter.getSortType());
+ ret.setStartIndex(0);
+ ret.setTotalCount(resultList.size());
+
+ ret.setPolicies(resultList);
+
+ return ret;
+ }
+
+ @Override
+ public RangerPolicyPaginatedList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getServicePolicies(serviceId, filter);
+
+ RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList();
+
+ ret.setResultSize(resultList.size());
+ ret.setPageSize(resultList.size());
+ ret.setSortBy(filter.getSortBy());
+ ret.setSortType(filter.getSortType());
+ ret.setStartIndex(0);
+ ret.setTotalCount(resultList.size());
+
+ ret.setPolicies(resultList);
+
+ return ret;
+ }
+
+ @Override
+ public RangerPolicyPaginatedList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getServicePolicies(serviceName, filter);
+
+ RangerPolicyPaginatedList ret = new RangerPolicyPaginatedList();
+
+ ret.setResultSize(resultList.size());
+ ret.setPageSize(resultList.size());
+ ret.setSortBy(filter.getSortBy());
+ ret.setSortType(filter.getSortType());
+ ret.setStartIndex(0);
+ ret.setTotalCount(resultList.size());
+
+ ret.setPolicies(resultList);
+
+ return ret;
+
+ }
+
+ @Override
+ public RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret;
+ }
+
+ @Override
+ public RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret;
+ }
+
+ @Override
+ public String getPolicyForVersionNumber(Long policyId) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret == null ? null : ret.getName();
+ }
+
+ protected void preCreate(RangerBaseModelObject obj) throws Exception {
+ obj.setId(new Long(0));
+ obj.setGuid(UUID.randomUUID().toString());
+ obj.setCreateTime(new Date());
+ obj.setUpdateTime(obj.getCreateTime());
+ obj.setVersion(new Long(1));
+ }
+
+ protected void preCreate(RangerService service) throws Exception {
+ preCreate((RangerBaseModelObject)service);
+
+ service.setPolicyVersion(new Long(0));
+ service.setPolicyUpdateTime(service.getCreateTime());
+ }
+
+ protected void postCreate(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ updateTagServiceDefForAddingAccessTypes((RangerServiceDef)obj);
+ }
+ }
+
+ protected void preUpdate(RangerBaseModelObject obj) throws Exception {
+ if(obj.getId() == null) {
+ obj.setId(new Long(0));
+ }
+
+ if(obj.getGuid() == null) {
+ obj.setGuid(UUID.randomUUID().toString());
+ }
+
+ if(obj.getCreateTime() == null) {
+ obj.setCreateTime(new Date());
+ }
+
+ Long version = obj.getVersion();
+
+ if(version == null) {
+ version = new Long(1);
+ } else {
+ version = new Long(version.longValue() + 1);
+ }
+
+ obj.setVersion(version);
+ obj.setUpdateTime(new Date());
+ }
+
+ protected void postUpdate(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ updateTagServiceDefForUpdatingAccessTypes((RangerServiceDef) obj);
+ }
+ }
+
+ protected void preDelete(RangerBaseModelObject obj) throws Exception {
+ // TODO:
+ }
+
+ protected void postDelete(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ updateTagServiceDefForDeletingAccessTypes(((RangerServiceDef) obj).getName());
+ }
+ }
+
+ protected long getMaxId(List<? extends RangerBaseModelObject> objs) {
+ long ret = -1;
+
+ if (objs != null) {
+ for (RangerBaseModelObject obj : objs) {
+ if (obj.getId() > ret) {
+ ret = obj.getId();
+ }
+ }
+ }
+ return ret;
+ }
+
+ private void updateTagServiceDefForAddingAccessTypes(RangerServiceDef serviceDef) throws Exception {
+ if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ return;
+ }
+
+ RangerServiceDef tagServiceDef = null;
+ try {
+ tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId());
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- Could not find TAG ServiceDef.. ", e);
+ throw e;
+ }
+ List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ for (RangerServiceDef.RangerAccessTypeDef accessType : serviceDef.getAccessTypes()) {
+ RangerServiceDef.RangerAccessTypeDef newAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType);
+
+ newAccessType.setItemId(serviceDef.getId()*(MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1) + accessType.getItemId());
+ newAccessType.setName(serviceDef.getName() + ":" + accessType.getName());
+ accessTypes.add(newAccessType);
+ }
+
+ tagServiceDef.getAccessTypes().addAll(accessTypes);
+ try {
+ updateServiceDef(tagServiceDef);
+ LOG.info("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- updated TAG service def with " + serviceDef.getName() + " access types");
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForAddingAccessTypes -- Failed to update TAG ServiceDef.. ", e);
+ throw e;
+ }
+ }
+
+ private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceDef) throws Exception {
+ if (serviceDef.getName().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ return;
+ }
+
+ String serviceDefName = serviceDef.getName();
+
+ RangerServiceDef tagServiceDef = null;
+ try {
+ tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId());
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Could not find TAG ServiceDef.. ", e);
+ throw e;
+ }
+
+ List<RangerServiceDef.RangerAccessTypeDef> tagSvcDefAccessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) {
+ if (accessType.getName().startsWith(serviceDefName + ":")) {
+ RangerServiceDef.RangerAccessTypeDef tagSvcDefAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType);
+ tagSvcDefAccessTypes.add(tagSvcDefAccessType);
+ }
+ }
+
+ List<RangerServiceDef.RangerAccessTypeDef> svcDefAccessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ for (RangerServiceDef.RangerAccessTypeDef accessType : serviceDef.getAccessTypes()) {
+ RangerServiceDef.RangerAccessTypeDef svcDefAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType);
+ svcDefAccessType.setItemId(serviceDef.getId()*(MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1) + accessType.getItemId());
+ svcDefAccessType.setName(serviceDefName + ":" + accessType.getName());
+ svcDefAccessTypes.add(svcDefAccessType);
+ }
+
+ tagServiceDef.getAccessTypes().removeAll(tagSvcDefAccessTypes);
+ tagServiceDef.getAccessTypes().addAll(svcDefAccessTypes);
+
+ try {
+ updateServiceDef(tagServiceDef);
+ LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- updated TAG service def with " + serviceDefName + " access types");
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- Failed to update TAG ServiceDef.. ", e);
+ throw e;
+ }
+
+ }
+
+ private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) throws Exception {
+ if (serviceDefName.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ return;
+ }
+
+ RangerServiceDef tagServiceDef = null;
+ try {
+ tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId());
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Could not find TAG ServiceDef.. ", e);
+ throw e;
+ }
+ List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) {
+ if (accessType.getName().startsWith(serviceDefName + ":")) {
+ RangerServiceDef.RangerAccessTypeDef newAccessType = new RangerServiceDef.RangerAccessTypeDef(accessType);
+ accessTypes.add(newAccessType);
+ }
+ }
+
+ tagServiceDef.getAccessTypes().removeAll(accessTypes);
+ try {
+ updateServiceDef(tagServiceDef);
+ LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types");
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Failed to update TAG ServiceDef.. ", e);
+ throw e;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java
new file mode 100644
index 0000000..9be733d
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractTagStore.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerService;
+
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+public abstract class AbstractTagStore implements TagStore {
+ protected void preCreate(RangerBaseModelObject obj) throws Exception {
+ obj.setId(new Long(0));
+ obj.setGuid(UUID.randomUUID().toString());
+ obj.setCreateTime(new Date());
+ obj.setUpdateTime(obj.getCreateTime());
+ obj.setVersion(new Long(1));
+ }
+
+ protected void postCreate(RangerBaseModelObject obj) throws Exception {
+ }
+
+ protected void preUpdate(RangerBaseModelObject obj) throws Exception {
+ if(obj.getId() == null) {
+ obj.setId(new Long(0));
+ }
+
+ if(obj.getGuid() == null) {
+ obj.setGuid(UUID.randomUUID().toString());
+ }
+
+ if(obj.getCreateTime() == null) {
+ obj.setCreateTime(new Date());
+ }
+
+ Long version = obj.getVersion();
+
+ if(version == null) {
+ version = new Long(1);
+ } else {
+ version = new Long(version.longValue() + 1);
+ }
+
+ obj.setVersion(version);
+ obj.setUpdateTime(new Date());
+ }
+
+ protected void postUpdate(RangerBaseModelObject obj) throws Exception {
+ }
+
+ protected void preDelete(RangerBaseModelObject obj) throws Exception {
+ // TODO:
+ }
+
+ protected void postDelete(RangerBaseModelObject obj) throws Exception {
+ }
+
+ protected long getMaxId(List<? extends RangerBaseModelObject> objs) {
+ long ret = -1;
+
+ if (objs != null) {
+ for (RangerBaseModelObject obj : objs) {
+ if (obj.getId() > ret) {
+ ret = obj.getId();
+ }
+ }
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
index 2930606..dcf6288 100755
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
@@ -43,6 +43,7 @@ public class EmbeddedServiceDefsUtil {
private static final Log LOG = LogFactory.getLog(EmbeddedServiceDefsUtil.class);
+ public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "_tag_";
public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs";
public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase";
public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive";
@@ -55,6 +56,7 @@ public class EmbeddedServiceDefsUtil {
private static EmbeddedServiceDefsUtil instance = new EmbeddedServiceDefsUtil();
private boolean createEmbeddedServiceDefs = true;
+ private RangerServiceDef tagServiceDef = null;
private RangerServiceDef hdfsServiceDef = null;
private RangerServiceDef hBaseServiceDef = null;
private RangerServiceDef hiveServiceDef = null;
@@ -82,6 +84,7 @@ public class EmbeddedServiceDefsUtil {
gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+ tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME);
hdfsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HDFS_NAME);
hBaseServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HBASE_NAME);
hiveServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_HIVE_NAME);
@@ -89,6 +92,9 @@ public class EmbeddedServiceDefsUtil {
stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME);
yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME);
kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME);
+
+ // Ensure that tag service def is updated with access types of all service defs
+ store.updateTagServiceDefForAccessTypes();
} catch(Throwable excp) {
LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp);
}
@@ -96,6 +102,8 @@ public class EmbeddedServiceDefsUtil {
LOG.info("<== EmbeddedServiceDefsUtil.init()");
}
+ public long getTagServiceDefId() { return getId(tagServiceDef); }
+
public long getHdfsServiceDefId() {
return getId(hdfsServiceDef);
}
@@ -141,7 +149,8 @@ public class EmbeddedServiceDefsUtil {
ret = loadEmbeddedServiceDef(serviceDefName);
LOG.info("creating embedded service-def " + serviceDefName);
- store.createServiceDef(ret);
+ ret = store.createServiceDef(ret);
+ LOG.info("created embedded service-def " + serviceDefName);
}
} catch(Exception excp) {
LOG.fatal("EmbeddedServiceDefsUtil.getOrCreateServiceDef(): failed to load/create serviceType " + serviceDefName, excp);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
new file mode 100644
index 0000000..8c3e0a9
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/PList.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import java.util.List;
+
+public abstract class PList implements java.io.Serializable {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Start index for the result
+ */
+ protected int startIndex;
+ /**
+ * Page size used for the result
+ */
+ protected int pageSize;
+ /**
+ * Total records in the database for the given search conditions
+ */
+ protected long totalCount;
+ /**
+ * Number of rows returned for the search condition
+ */
+ protected int resultSize;
+ /**
+ * Sort type. Either desc or asc
+ */
+ protected String sortType;
+ /**
+ * Comma seperated list of the fields for sorting
+ */
+ protected String sortBy;
+
+ protected long queryTimeMS = System.currentTimeMillis();
+
+
+ /**
+ * Default constructor. This will set all the attributes to default value.
+ */
+ public PList() {
+ }
+
+ /**
+ * Initialize with existing list
+ *
+ * @param objectList
+ */
+ public PList(@SuppressWarnings("rawtypes") List objectList) {
+ int size = 0;
+ if (objectList != null) {
+ size = objectList.size();
+ }
+
+ startIndex = 0;
+ pageSize = size;
+ totalCount = size;
+ resultSize = size;
+ sortType = null;
+ sortBy = null;
+ }
+
+ abstract public int getListSize();
+
+ abstract public List<?> getList();
+
+ /**
+ * This method sets the value to the member attribute <b>startIndex</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param startIndex
+ * Value to set member attribute <b>startIndex</b>
+ */
+ public void setStartIndex(int startIndex) {
+ this.startIndex = startIndex;
+ }
+ public int getStartIndex() { return startIndex; }
+
+
+ /**
+ * This method sets the value to the member attribute <b>pageSize</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param pageSize
+ * Value to set member attribute <b>pageSize</b>
+ */
+ public void setPageSize(int pageSize) {
+ this.pageSize = pageSize;
+ }
+ public int getPageSize() { return pageSize; }
+
+
+ /**
+ * This method sets the value to the member attribute <b>totalCount</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param totalCount
+ * Value to set member attribute <b>totalCount</b>
+ */
+ public void setTotalCount(long totalCount) {
+ this.totalCount = totalCount;
+ }
+ public long getTotalCount() { return totalCount; }
+
+
+
+ /**
+ * This method sets the value to the member attribute <b>resultSize</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param resultSize
+ * Value to set member attribute <b>resultSize</b>
+ */
+ public void setResultSize(int resultSize) {
+ this.resultSize = resultSize;
+ }
+
+ /**
+ * Returns the value for the member attribute <b>resultSize</b>
+ *
+ * @return int - value of member attribute <b>resultSize</b>.
+ */
+ public int getResultSize() {
+ return getListSize();
+ }
+
+ /**
+ * This method sets the value to the member attribute <b>sortType</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param sortType
+ * Value to set member attribute <b>sortType</b>
+ */
+ public void setSortType(String sortType) {
+ this.sortType = sortType;
+ }
+ public String getSortType() { return sortType; }
+
+
+
+ /**
+ * This method sets the value to the member attribute <b>sortBy</b>. You
+ * cannot set null to the attribute.
+ *
+ * @param sortBy
+ * Value to set member attribute <b>sortBy</b>
+ */
+ public void setSortBy(String sortBy) {
+ this.sortBy = sortBy;
+ }
+ public String getSortBy() { return sortBy; }
+
+
+
+
+
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ return "PList [startIndex=" + startIndex + ", pageSize="
+ + pageSize + ", totalCount=" + totalCount
+ + ", resultSize=" + resultSize + ", sortType="
+ + sortType + ", sortBy=" + sortBy + ", queryTimeMS="
+ + queryTimeMS + "]";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java
new file mode 100644
index 0000000..9573886
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerPolicyPaginatedList.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RangerPolicyPaginatedList extends PList {
+ private static final long serialVersionUID = 1L;
+
+ List<RangerPolicy> policies = new ArrayList<RangerPolicy>();
+
+ public RangerPolicyPaginatedList() {
+ super();
+ }
+
+ public RangerPolicyPaginatedList(List<RangerPolicy> objList) {
+ super(objList);
+ this.policies = objList;
+ }
+
+ public List<RangerPolicy> getPolicies() {
+ return policies;
+ }
+
+ public void setPolicies(List<RangerPolicy> policies) {
+ this.policies = policies;
+ }
+
+ @Override
+ public int getListSize() {
+ if (policies != null) {
+ return policies.size();
+ }
+ return 0;
+ }
+
+ @Override
+ public List<?> getList() {
+ return policies;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java
new file mode 100644
index 0000000..7366ac8
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServiceDefPaginatedList.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.ranger.plugin.model.RangerServiceDef;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RangerServiceDefPaginatedList extends PList {
+ private static final long serialVersionUID = 1L;
+
+ List<RangerServiceDef> serviceDefs = new ArrayList<RangerServiceDef>();
+
+ public RangerServiceDefPaginatedList() {
+ super();
+ }
+
+ public RangerServiceDefPaginatedList(List<RangerServiceDef> objList) {
+ super(objList);
+ this.serviceDefs = objList;
+ }
+
+ public List<RangerServiceDef> getServiceDefs() {
+ return serviceDefs;
+ }
+
+ public void setServiceDefs(List<RangerServiceDef> serviceDefs) {
+ this.serviceDefs = serviceDefs;
+ }
+
+ @Override
+ public int getListSize() {
+ if (serviceDefs != null) {
+ return serviceDefs.size();
+ }
+ return 0;
+ }
+
+ @Override
+ public List<?> getList() {
+ return serviceDefs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java
new file mode 100644
index 0000000..6194bb8
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/RangerServicePaginatedList.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RangerServicePaginatedList extends PList {
+ private static final long serialVersionUID = 1L;
+
+ List<RangerService> services = new ArrayList<RangerService>();
+
+ public RangerServicePaginatedList() {
+ super();
+ }
+
+ public RangerServicePaginatedList(List<RangerService> objList) {
+ super(objList);
+ this.services = objList;
+ }
+
+ public List<RangerService> getServices() {
+ return services;
+ }
+
+ public void setServices(List<RangerService> services) {
+ this.services = services;
+ }
+
+ @Override
+ public int getListSize() {
+ if (services != null) {
+ return services.size();
+ }
+ return 0;
+ }
+
+ @Override
+ public List<?> getList() {
+ return services;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
index b998e93..0915cf5 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/ServiceStore.java
@@ -35,6 +35,10 @@ public interface ServiceStore {
RangerServiceDef updateServiceDef(RangerServiceDef serviceDef) throws Exception;
void deleteServiceDef(Long id) throws Exception;
+ void deleteServiceDef(Long id, Boolean forceDelete) throws Exception;
+
+ void updateTagServiceDefForAccessTypes() throws Exception;
+
RangerServiceDef getServiceDef(Long id) throws Exception;
@@ -42,6 +46,7 @@ public interface ServiceStore {
List<RangerServiceDef> getServiceDefs(SearchFilter filter) throws Exception;
+ RangerServiceDefPaginatedList getPaginatedServiceDefs(SearchFilter filter) throws Exception;
RangerService createService(RangerService service) throws Exception;
@@ -55,6 +60,7 @@ public interface ServiceStore {
List<RangerService> getServices(SearchFilter filter) throws Exception;
+ RangerServicePaginatedList getPaginatedServices(SearchFilter filter) throws Exception;
RangerPolicy createPolicy(RangerPolicy policy) throws Exception;
@@ -66,11 +72,25 @@ public interface ServiceStore {
List<RangerPolicy> getPolicies(SearchFilter filter) throws Exception;
+ RangerPolicyPaginatedList getPaginatedPolicies(SearchFilter filter) throws Exception;
+
List<RangerPolicy> getPoliciesByResourceSignature(String serviceName, String policySignature, Boolean isPolicyEnabled) throws Exception;
List<RangerPolicy> getServicePolicies(Long serviceId, SearchFilter filter) throws Exception;
+ RangerPolicyPaginatedList getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception;
+
List<RangerPolicy> getServicePolicies(String serviceName, SearchFilter filter) throws Exception;
+ RangerPolicyPaginatedList getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception;
+
ServicePolicies getServicePoliciesIfUpdated(String serviceName, Long lastKnownVersion) throws Exception;
-}
+
+ RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId);
+
+ RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo);
+
+ String getPolicyForVersionNumber(Long policyId);
+
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java
new file mode 100644
index 0000000..b880179
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagPredicateUtil.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.Predicate;
+import org.apache.commons.collections.PredicateUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerResource;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.util.SearchFilter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TagPredicateUtil extends AbstractPredicateUtil {
+ private TagStore tagStore = null;
+ public TagPredicateUtil(TagStore tagStore) {
+ super();
+ this.tagStore = tagStore;
+ }
+
+ @Override
+ public void addPredicates(SearchFilter filter, List<Predicate> predicates) {
+ super.addPredicates(filter, predicates);
+
+ addPredicateForTagDefId(filter.getParam(SearchFilter.TAG_DEF_ID), predicates);
+ addPredicateForTagDefName(filter.getParam(SearchFilter.TAG_DEF_NAME), predicates);
+
+ addPredicateForTagResourceServiceName(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME), predicates);
+ addPredicateForTagResourceServiceType(filter.getParam(SearchFilter.TAG_RESOURCE_SERVICE_TYPE), predicates);
+
+ addPredicateForTagResourceId(filter.getParam(SearchFilter.TAG_RESOURCE_ID), predicates);
+ }
+
+ private Predicate addPredicateForTagDefId(final String id, List<Predicate> predicates) {
+ if (StringUtils.isEmpty(id)) {
+ return null;
+ }
+
+ Predicate ret = new Predicate() {
+ @Override
+ public boolean evaluate(Object object) {
+
+ boolean ret = false;
+
+ if (object == null) {
+ return ret;
+ }
+
+ if (object instanceof RangerTagDef) {
+ RangerTagDef tagDef = (RangerTagDef) object;
+
+ ret = StringUtils.equals(id, tagDef.getId().toString());
+ }
+
+ return ret;
+ }
+ };
+
+ if (predicates != null) {
+ predicates.add(ret);
+ }
+
+ return ret;
+ }
+
+ private Predicate addPredicateForTagDefName(final String name, List<Predicate> predicates) {
+ if (name == null || StringUtils.isEmpty(name)) {
+ return null;
+ }
+
+ Predicate ret = new Predicate() {
+ @Override
+ public boolean evaluate(Object object) {
+
+ boolean ret = false;
+
+ if (object == null) {
+ return ret;
+ }
+
+ if (object instanceof RangerTagDef) {
+ RangerTagDef tagDef = (RangerTagDef) object;
+
+ ret = StringUtils.equals(name, tagDef.getName());
+ }
+
+ return ret;
+ }
+ };
+
+ if (predicates != null) {
+ predicates.add(ret);
+ }
+
+ return ret;
+ }
+
+ private Predicate addPredicateForTagResourceServiceName(final String name, List<Predicate> predicates) {
+ if (name == null || StringUtils.isEmpty(name)) {
+ return null;
+ }
+
+ Predicate ret = new Predicate() {
+ @Override
+ public boolean evaluate(Object object) {
+
+ boolean ret = false;
+
+ if (object == null) {
+ return ret;
+ }
+
+ if (object instanceof RangerResource) {
+ RangerResource rangerResource = (RangerResource) object;
+
+ ret = StringUtils.equals(name, rangerResource.getTagServiceName());
+ }
+
+ return ret;
+ }
+ };
+
+ if (predicates != null) {
+ predicates.add(ret);
+ }
+
+ return ret;
+ }
+
+ private Predicate addPredicateForTagResourceServiceType(final String type, List<Predicate> predicates) {
+ if (type == null || StringUtils.isEmpty(type)) {
+ return null;
+ }
+
+ Predicate ret = new Predicate() {
+ @Override
+ public boolean evaluate(Object object) {
+
+ boolean ret = false;
+
+ if (object == null) {
+ return ret;
+ }
+
+ if (object instanceof RangerResource) {
+ RangerResource rangerResource = (RangerResource) object;
+
+ ret = StringUtils.equals(type, rangerResource.getServiceType());
+ }
+
+ return ret;
+ }
+ };
+
+ if (predicates != null) {
+ predicates.add(ret);
+ }
+
+ return ret;
+ }
+ private Predicate addPredicateForTagResourceId(final String id, List<Predicate> predicates) {
+ if (StringUtils.isEmpty(id)) {
+ return null;
+ }
+
+ Predicate ret = new Predicate() {
+ @Override
+ public boolean evaluate(Object object) {
+
+ boolean ret = false;
+
+ if (object == null) {
+ return ret;
+ }
+
+ if (object instanceof RangerResource) {
+ RangerResource rangerResource = (RangerResource) object;
+
+ ret = StringUtils.equals(id, rangerResource.getId().toString());
+ }
+
+ return ret;
+ }
+ };
+
+ if (predicates != null) {
+ predicates.add(ret);
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
new file mode 100644
index 0000000..87e2b6f
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.ranger.plugin.model.RangerResource;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.util.SearchFilter;
+
+import java.util.List;
+
+/**
+ * Interface to backing store for the top-level TAG model objects
+ */
+
+public interface TagStore {
+ void init() throws Exception;
+
+ RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception;
+
+ RangerTagDef updateTagDef(RangerTagDef TagDef) throws Exception;
+
+ void deleteTagDef(String name) throws Exception;
+
+ RangerTagDef getTagDef(String name) throws Exception;
+
+ RangerTagDef getTagDefById(Long id) throws Exception;
+
+ List<RangerTagDef> getTagDefs(SearchFilter filter) throws Exception;
+
+
+ RangerResource createResource(RangerResource resource) throws Exception;
+
+ RangerResource updateResource(RangerResource resource) throws Exception;
+
+ void deleteResource(Long id) throws Exception;
+
+ RangerResource getResource(Long id) throws Exception;
+
+ List<RangerResource> getResources(String tagServiceName, String serviceType) throws Exception;
+
+ List<RangerResource> getResources(SearchFilter filter) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java
deleted file mode 100644
index 6ce2589..0000000
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/BaseFileStore.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ranger.plugin.store.file;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
-import org.apache.ranger.plugin.model.RangerBaseModelObject;
-import org.apache.ranger.plugin.model.RangerPolicy;
-import org.apache.ranger.plugin.model.RangerService;
-import org.apache.ranger.plugin.model.RangerServiceDef;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-public class BaseFileStore {
- private static final Log LOG = LogFactory.getLog(BaseFileStore.class);
-
- private Gson gsonBuilder = null;
- private String dataDir = null;
-
- protected static final String FILE_PREFIX_SERVICE_DEF = "ranger-servicedef-";
- protected static final String FILE_PREFIX_SERVICE = "ranger-service-";
- protected static final String FILE_PREFIX_POLICY = "ranger-policy-";
- protected static final String FILE_SUFFIX_JSON = ".json";
-
- protected void initStore(String dataDir) {
- this.dataDir = dataDir;
-
- try {
- gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
- } catch(Throwable excp) {
- LOG.fatal("BaseFileStore.init(): failed to create GsonBuilder object", excp);
- }
- }
-
- protected String getDataDir() {
- return dataDir;
- }
-
- protected String getServiceDefFile(Long id) {
- String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_SERVICE_DEF + id + FILE_SUFFIX_JSON;
-
- return filePath;
- }
-
- protected String getServiceFile(Long id) {
- String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_SERVICE + id + FILE_SUFFIX_JSON;
-
- return filePath;
- }
-
- protected String getPolicyFile(Long serviceId, Long policyId) {
- String filePath = dataDir + Path.SEPARATOR + FILE_PREFIX_POLICY + serviceId + "-" + policyId + FILE_SUFFIX_JSON;
-
- return filePath;
- }
-
- protected <T> T loadFromResource(String resource, Class<T> cls) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.loadFromResource(" + resource + ")");
- }
-
- InputStream inStream = this.getClass().getResourceAsStream(resource);
-
- T ret = loadFromStream(inStream, cls);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.loadFromResource(" + resource + "): " + ret);
- }
-
- return ret;
- }
-
- protected <T> T loadFromStream(InputStream inStream, Class<T> cls) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.loadFromStream()");
- }
-
- InputStreamReader reader = new InputStreamReader(inStream);
-
- T ret = gsonBuilder.fromJson(reader, cls);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.loadFromStream(): " + ret);
- }
-
- return ret;
- }
-
- protected <T> T loadFromFile(Path filePath, Class<T> cls) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.loadFromFile(" + filePath + ")");
- }
-
- T ret = null;
- InputStreamReader reader = null;
-
- try {
- FileSystem fileSystem = getFileSystem(filePath);
- FSDataInputStream inStream = fileSystem.open(filePath);
-
- ret = loadFromStream(inStream, cls);
- } finally {
- close(reader);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.loadFromFile(" + filePath + "): " + ret);
- }
-
- return ret;
- }
-
- protected <T> List<T> loadFromDir(Path dirPath, final String filePrefix, Class<T> cls) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.loadFromDir()");
- }
-
- List<T> ret = new ArrayList<T>();
-
- try {
- FileSystem fileSystem = getFileSystem(dirPath);
-
- if(fileSystem.exists(dirPath) && fileSystem.isDirectory(dirPath)) {
- PathFilter filter = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return path.getName().startsWith(filePrefix) &&
- path.getName().endsWith(FILE_SUFFIX_JSON);
- }
- };
-
- FileStatus[] sdFiles = fileSystem.listStatus(dirPath, filter);
-
- if(sdFiles != null) {
- for(FileStatus sdFile : sdFiles) {
- T obj = loadFromFile(sdFile.getPath(), cls);
-
- if(obj != null) {
- ret.add(obj);
- }
- }
- }
- } else {
- LOG.error(dirPath + ": does not exists or not a directory");
- }
- } catch(IOException excp) {
- LOG.warn("error loading service-def in directory " + dirPath, excp);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.loadFromDir(): count=" + (ret == null ? 0 : ret.size()));
- }
-
- return ret;
- }
-
- protected <T> T saveToFile(T obj, Path filePath, boolean overWrite) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.saveToFile(" + filePath + ")");
- }
-
- OutputStreamWriter writer = null;
-
- try {
- FileSystem fileSystem = getFileSystem(filePath);
- FSDataOutputStream outStream = fileSystem.create(filePath, overWrite);
-
- writer = new OutputStreamWriter(outStream);
-
- gsonBuilder.toJson(obj, writer);
- } finally {
- close(writer);
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.saveToFile(" + filePath + "): " + obj);
- }
-
- return obj;
- }
-
- protected boolean deleteFile(Path filePath) throws Exception {
- LOG.debug("==> BaseFileStore.deleteFile(" + filePath + ")");
-
- FileSystem fileSystem = getFileSystem(filePath);
-
- boolean ret = false;
-
- if(fileSystem.exists(filePath)) {
- ret = fileSystem.delete(filePath, false);
- } else {
- ret = true; // nothing to delete
- }
-
- LOG.debug("<== BaseFileStore.deleteFile(" + filePath + "): " + ret);
-
- return ret;
- }
-
- protected boolean renamePath(Path oldPath, Path newPath) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.renamePath(" + oldPath + "," + newPath + ")");
- }
-
- FileSystem fileSystem = getFileSystem(oldPath);
-
- boolean ret = false;
-
- if(fileSystem.exists(oldPath)) {
- if(! fileSystem.exists(newPath)) {
- ret = fileSystem.rename(oldPath, newPath);
- } else {
- LOG.warn("target of rename '" + newPath + "' already exists");
- }
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.renamePath(" + oldPath + "," + newPath + "): " + ret);
- }
-
- return ret;
- }
-
- protected RangerServiceDef saveToFile(RangerServiceDef serviceDef, boolean overWrite) throws Exception {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + ")");
- }
-
- Path filePath = new Path(getServiceDefFile(serviceDef.getId()));
-
- RangerServiceDef ret = saveToFile(serviceDef, filePath, overWrite);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== BaseFileStore.saveToFile(" + serviceDef + "," + overWrite + "): ");
- }
-
- return ret;
- }
-
- protected RangerService saveToFile(RangerService service, boolean overWrite) throws Exception {
- Path filePath = new Path(getServiceFile(service.getId()));
-
- RangerService ret = saveToFile(service, filePath, overWrite);
-
- return ret;
- }
-
- protected RangerPolicy saveToFile(RangerPolicy policy, long serviceId, boolean overWrite) throws Exception {
- Path filePath = new Path(getPolicyFile(serviceId, policy.getId()));
-
- RangerPolicy ret = saveToFile(policy, filePath, overWrite);
-
- return ret;
- }
-
- protected long getMaxId(List<? extends RangerBaseModelObject> objs) {
- long ret = -1;
-
- if(objs != null) {
- for(RangerBaseModelObject obj : objs) {
- if(obj.getId() > ret) {
- ret = obj.getId();
- }
- }
- }
-
- return ret;
- }
- protected FileSystem getFileSystem(Path filePath) throws Exception {
- Configuration conf = new Configuration();
- FileSystem fileSystem = filePath.getFileSystem(conf);
-
- return fileSystem;
- }
-
- protected void close(FileSystem fs) {
- if(fs != null) {
- try {
- fs.close();
- } catch(IOException excp) {
- // ignore
- }
- }
- }
-
- protected void close(InputStreamReader reader) {
- if(reader != null) {
- try {
- reader.close();
- } catch(IOException excp) {
- // ignore
- }
- }
- }
-
- protected void close(OutputStreamWriter writer) {
- if(writer != null) {
- try {
- writer.close();
- } catch(IOException excp) {
- // ignore
- }
- }
- }
-
- protected void preCreate(RangerBaseModelObject obj) {
- obj.setId(new Long(0));
- obj.setGuid(UUID.randomUUID().toString());
- obj.setCreateTime(new Date());
- obj.setUpdateTime(obj.getCreateTime());
- obj.setVersion(new Long(1));
- }
-
- protected void preCreate(RangerService service) {
- preCreate((RangerBaseModelObject)service);
-
- service.setPolicyVersion(new Long(0));
- service.setPolicyUpdateTime(service.getCreateTime());
- }
-
- protected void postCreate(RangerBaseModelObject obj) {
- // TODO:
- }
-
- protected void preUpdate(RangerBaseModelObject obj) {
- if(obj.getId() == null) {
- obj.setId(new Long(0));
- }
-
- if(obj.getGuid() == null) {
- obj.setGuid(UUID.randomUUID().toString());
- }
-
- if(obj.getCreateTime() == null) {
- obj.setCreateTime(new Date());
- }
-
- Long version = obj.getVersion();
-
- if(version == null) {
- version = new Long(1);
- } else {
- version = new Long(version.longValue() + 1);
- }
-
- obj.setVersion(version);
- obj.setUpdateTime(new Date());
- }
-
- protected void postUpdate(RangerBaseModelObject obj) {
- // TODO:
- }
-
- protected void preDelete(RangerBaseModelObject obj) {
- // TODO:
- }
-
- protected void postDelete(RangerBaseModelObject obj) {
- // TODO:
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java
new file mode 100644
index 0000000..50de24b
--- /dev/null
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/FileStoreUtil.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store.file;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+public class FileStoreUtil {
+ private static final Log LOG = LogFactory.getLog(FileStoreUtil.class);
+
+ private Gson gsonBuilder = null;
+ private String dataDir = null;
+
+ private static final String FILE_SUFFIX_JSON = ".json";
+
+ public void initStore(String dataDir) {
+ this.dataDir = dataDir;
+
+ try {
+ gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").setPrettyPrinting().create();
+ } catch(Throwable excp) {
+ LOG.fatal("FileStoreUtil.init(): failed to create GsonBuilder object", excp);
+ }
+ }
+
+ public String getDataDir() {
+ return dataDir;
+ }
+
+ public String getDataFile(String filePrefix, Long id) {
+ String filePath = dataDir + Path.SEPARATOR + filePrefix + id + FILE_SUFFIX_JSON;
+
+ return filePath;
+ }
+
+ public String getDataFile(String filePrefix, Long parentId, Long objectId) {
+ String filePath = dataDir + Path.SEPARATOR + filePrefix + parentId + "-" + objectId + FILE_SUFFIX_JSON;
+
+ return filePath;
+ }
+
+ public <T> T loadFromResource(String resource, Class<T> cls) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.loadFromResource(" + resource + ")");
+ }
+
+ InputStream inStream = this.getClass().getResourceAsStream(resource);
+
+ T ret = loadFromStream(inStream, cls);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.loadFromResource(" + resource + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ public <T> T loadFromStream(InputStream inStream, Class<T> cls) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.loadFromStream()");
+ }
+
+ InputStreamReader reader = new InputStreamReader(inStream);
+
+ T ret = gsonBuilder.fromJson(reader, cls);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.loadFromStream(): " + ret);
+ }
+
+ return ret;
+ }
+
+ public <T> T loadFromFile(Path filePath, Class<T> cls) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.loadFromFile(" + filePath + ")");
+ }
+
+ T ret = null;
+ InputStreamReader reader = null;
+
+ try {
+ FileSystem fileSystem = getFileSystem(filePath);
+ FSDataInputStream inStream = fileSystem.open(filePath);
+
+ ret = loadFromStream(inStream, cls);
+ } finally {
+ close(reader);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.loadFromFile(" + filePath + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ public <T> List<T> loadFromDir(Path dirPath, final String filePrefix, Class<T> cls) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.loadFromDir()");
+ }
+
+ List<T> ret = new ArrayList<T>();
+
+ try {
+ FileSystem fileSystem = getFileSystem(dirPath);
+
+ if(fileSystem.exists(dirPath) && fileSystem.isDirectory(dirPath)) {
+ PathFilter filter = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith(filePrefix) &&
+ path.getName().endsWith(FILE_SUFFIX_JSON);
+ }
+ };
+
+ FileStatus[] sdFiles = fileSystem.listStatus(dirPath, filter);
+
+ if(sdFiles != null) {
+ for(FileStatus sdFile : sdFiles) {
+ T obj = loadFromFile(sdFile.getPath(), cls);
+
+ if(obj != null) {
+ ret.add(obj);
+ }
+ }
+ }
+ } else {
+ LOG.error(dirPath + ": does not exists or not a directory");
+ }
+ } catch(IOException excp) {
+ LOG.warn("error loading service-def in directory " + dirPath, excp);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.loadFromDir(): count=" + (ret == null ? 0 : ret.size()));
+ }
+
+ return ret;
+ }
+
+ public <T> T saveToFile(T obj, Path filePath, boolean overWrite) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.saveToFile(" + filePath + ")");
+ }
+
+ OutputStreamWriter writer = null;
+
+ try {
+ FileSystem fileSystem = getFileSystem(filePath);
+ FSDataOutputStream outStream = fileSystem.create(filePath, overWrite);
+
+ writer = new OutputStreamWriter(outStream);
+
+ gsonBuilder.toJson(obj, writer);
+ } finally {
+ close(writer);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.saveToFile(" + filePath + "): " + obj);
+ }
+
+ return obj;
+ }
+
+ public boolean deleteFile(Path filePath) throws Exception {
+ LOG.debug("==> FileStoreUtil.deleteFile(" + filePath + ")");
+
+ FileSystem fileSystem = getFileSystem(filePath);
+
+ boolean ret = false;
+
+ if(fileSystem.exists(filePath)) {
+ ret = fileSystem.delete(filePath, false);
+ } else {
+ ret = true; // nothing to delete
+ }
+
+ LOG.debug("<== FileStoreUtil.deleteFile(" + filePath + "): " + ret);
+
+ return ret;
+ }
+
+ public boolean renamePath(Path oldPath, Path newPath) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.renamePath(" + oldPath + "," + newPath + ")");
+ }
+
+ FileSystem fileSystem = getFileSystem(oldPath);
+
+ boolean ret = false;
+
+ if(fileSystem.exists(oldPath)) {
+ if(! fileSystem.exists(newPath)) {
+ ret = fileSystem.rename(oldPath, newPath);
+ } else {
+ LOG.warn("target of rename '" + newPath + "' already exists");
+ }
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.renamePath(" + oldPath + "," + newPath + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ public RangerServiceDef saveToFile(RangerServiceDef serviceDef, String filePrefix, boolean overWrite) throws Exception {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> FileStoreUtil.saveToFile(" + serviceDef + "," + overWrite + ")");
+ }
+
+ Path filePath = new Path(getDataFile(filePrefix, serviceDef.getId()));
+
+ RangerServiceDef ret = saveToFile(serviceDef, filePath, overWrite);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== FileStoreUtil.saveToFile(" + serviceDef + "," + overWrite + "): ");
+ }
+
+ return ret;
+ }
+
+ public RangerService saveToFile(RangerService service, String filePrefix,boolean overWrite) throws Exception {
+ Path filePath = new Path(getDataFile(filePrefix, service.getId()));
+
+ RangerService ret = saveToFile(service, filePath, overWrite);
+
+ return ret;
+ }
+
+ public RangerPolicy saveToFile(RangerPolicy policy, String filePrefix, long serviceId, boolean overWrite) throws Exception {
+ Path filePath = new Path(getDataFile(filePrefix, serviceId, policy.getId()));
+
+ RangerPolicy ret = saveToFile(policy, filePath, overWrite);
+
+ return ret;
+ }
+
+ public FileSystem getFileSystem(Path filePath) throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fileSystem = filePath.getFileSystem(conf);
+
+ return fileSystem;
+ }
+
+ protected void close(FileSystem fs) {
+ if(fs != null) {
+ try {
+ fs.close();
+ } catch(IOException excp) {
+ // ignore
+ }
+ }
+ }
+
+ protected void close(InputStreamReader reader) {
+ if(reader != null) {
+ try {
+ reader.close();
+ } catch(IOException excp) {
+ // ignore
+ }
+ }
+ }
+
+ protected void close(OutputStreamWriter writer) {
+ if(writer != null) {
+ try {
+ writer.close();
+ } catch(IOException excp) {
+ // ignore
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/9578f94c/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
index 2e469cd..8a34c80 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/ServiceFileStore.java
@@ -36,32 +36,39 @@ import org.apache.ranger.plugin.model.RangerBaseModelObject;
import org.apache.ranger.plugin.model.RangerPolicy;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.store.AbstractServiceStore;
import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
import org.apache.ranger.plugin.store.ServicePredicateUtil;
import org.apache.ranger.plugin.store.ServiceStore;
+import org.apache.ranger.plugin.store.file.FileStoreUtil;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServicePolicies;
-
-public class ServiceFileStore extends BaseFileStore implements ServiceStore {
+public class ServiceFileStore extends AbstractServiceStore {
private static final Log LOG = LogFactory.getLog(ServiceFileStore.class);
public static final String PROPERTY_SERVICE_FILE_STORE_DIR = "ranger.service.store.file.dir";
+ protected static final String FILE_PREFIX_SERVICE_DEF = "ranger-servicedef-";
+ protected static final String FILE_PREFIX_SERVICE = "ranger-service-";
+ protected static final String FILE_PREFIX_POLICY = "ranger-policy-";
+
private String dataDir = null;
private long nextServiceDefId = 0;
private long nextServiceId = 0;
private long nextPolicyId = 0;
private ServicePredicateUtil predicateUtil = null;
+ private FileStoreUtil fileStoreUtil = null;
public ServiceFileStore() {
if(LOG.isDebugEnabled()) {
LOG.debug("==> ServiceFileStore.ServiceFileStore()");
}
- dataDir = RangerConfiguration.getInstance().get(PROPERTY_SERVICE_FILE_STORE_DIR, "file:///etc/ranger/data");
+ this.dataDir = RangerConfiguration.getInstance().get(PROPERTY_SERVICE_FILE_STORE_DIR, "file:///etc/ranger/data");
predicateUtil = new ServicePredicateUtil(this);
+ fileStoreUtil = new FileStoreUtil();
if(LOG.isDebugEnabled()) {
LOG.debug("<== ServiceFileStore.ServiceFileStore()");
@@ -75,6 +82,8 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
this.dataDir = dataDir;
predicateUtil = new ServicePredicateUtil(this);
+ fileStoreUtil = new FileStoreUtil();
+ fileStoreUtil.initStore(dataDir);
if(LOG.isDebugEnabled()) {
LOG.debug("<== ServiceFileStore.ServiceFileStore()");
@@ -87,7 +96,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
LOG.debug("==> ServiceFileStore.init()");
}
- super.initStore(dataDir);
+ fileStoreUtil.initStore(dataDir);
EmbeddedServiceDefsUtil.instance().init(this);
@@ -115,7 +124,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
serviceDef.setId(nextServiceDefId++);
- ret = saveToFile(serviceDef, false);
+ ret = fileStoreUtil.saveToFile(serviceDef, FILE_PREFIX_SERVICE_DEF, false);
postCreate(ret);
} catch(Exception excp) {
@@ -161,7 +170,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
preUpdate(existing);
- ret = saveToFile(existing, true);
+ ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_SERVICE_DEF, true);
postUpdate(ret);
} catch(Exception excp) {
@@ -194,9 +203,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
try {
preDelete(existing);
- Path filePath = new Path(getServiceDefFile(id));
+ Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE_DEF, id));
- deleteFile(filePath);
+ fileStoreUtil.deleteFile(filePath);
postDelete(existing);
} catch(Exception excp) {
@@ -299,7 +308,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
service.setId(nextServiceId++);
- ret = saveToFile(service, false);
+ ret = fileStoreUtil.saveToFile(service, FILE_PREFIX_SERVICE, false);
postCreate(service);
} catch(Exception excp) {
@@ -344,7 +353,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
preUpdate(existing);
- ret = saveToFile(existing, true);
+ ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_SERVICE, true);
postUpdate(ret);
@@ -375,13 +384,13 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
}
try {
- Path filePath = new Path(getServiceFile(id));
+ Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE, id));
preDelete(existing);
handleServiceDelete(existing);
- deleteFile(filePath);
+ fileStoreUtil.deleteFile(filePath);
postDelete(existing);
} catch(Exception excp) {
@@ -402,9 +411,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
RangerService ret = null;
try {
- Path filePath = new Path(getServiceFile(id));
+ Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_SERVICE, id));
- ret = loadFromFile(filePath, RangerService.class);
+ ret = fileStoreUtil.loadFromFile(filePath, RangerService.class);
} catch(Exception excp) {
LOG.error("ServiceFileStore.getService(" + id + "): failed to read service", excp);
}
@@ -489,7 +498,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
policy.setId(nextPolicyId++);
- ret = saveToFile(policy, service.getId(), false);
+ ret = fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), false);
handlePolicyUpdate(service);
@@ -544,7 +553,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
preUpdate(existing);
- ret = saveToFile(existing, service.getId(), true);
+ ret = fileStoreUtil.saveToFile(existing, FILE_PREFIX_POLICY, service.getId(), true);
handlePolicyUpdate(service);
@@ -581,9 +590,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
try {
preDelete(existing);
- Path filePath = new Path(getPolicyFile(service.getId(), existing.getId()));
+ Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_POLICY, service.getId(), existing.getId()));
- deleteFile(filePath);
+ fileStoreUtil.deleteFile(filePath);
handlePolicyUpdate(service);
@@ -768,7 +777,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
preUpdate(policy);
- saveToFile(policy, service.getId(), true);
+ fileStoreUtil.saveToFile(policy, FILE_PREFIX_POLICY, service.getId(), true);
postUpdate(policy);
}
@@ -787,9 +796,9 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
preDelete(policy);
- Path filePath = new Path(getPolicyFile(service.getId(), policy.getId()));
+ Path filePath = new Path(fileStoreUtil.getDataFile(FILE_PREFIX_POLICY, service.getId(), policy.getId()));
- deleteFile(filePath);
+ fileStoreUtil.deleteFile(filePath);
postDelete(policy);
}
@@ -812,7 +821,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
service.setPolicyVersion(policyVersion);
service.setPolicyUpdateTime(new Date());
- saveToFile(service, true);
+ fileStoreUtil.saveToFile(service, FILE_PREFIX_SERVICE, true);
}
private RangerPolicy findPolicyByName(String serviceName, String policyName) throws Exception {
@@ -853,7 +862,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
try {
// load service definitions from file system
- List<RangerServiceDef> sds = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class);
+ List<RangerServiceDef> sds = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_SERVICE_DEF, RangerServiceDef.class);
if(sds != null) {
for(RangerServiceDef sd : sds) {
@@ -900,7 +909,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
List<RangerService> ret = null;
try {
- ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_SERVICE, RangerService.class);
+ ret = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_SERVICE, RangerService.class);
nextServiceId = getMaxId(ret) + 1;
} catch(Exception excp) {
@@ -926,7 +935,7 @@ public class ServiceFileStore extends BaseFileStore implements ServiceStore {
List<RangerPolicy> ret = null;
try {
- ret = loadFromDir(new Path(getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class);
+ ret = fileStoreUtil.loadFromDir(new Path(fileStoreUtil.getDataDir()), FILE_PREFIX_POLICY, RangerPolicy.class);
nextPolicyId = getMaxId(ret) + 1;
} catch(Exception excp) {