You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/06/03 05:12:02 UTC
[3/3] incubator-ranger git commit: Merge branch 'master' into
tag-policy
Merge branch 'master' into tag-policy
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/4915b823
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/4915b823
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/4915b823
Branch: refs/heads/tag-policy
Commit: 4915b823fa247b6fe6c1035a9ec15c7cfec5571c
Parents: 5d437fa 4c45671
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Jun 2 20:11:52 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jun 2 20:11:52 2015 -0700
----------------------------------------------------------------------
.../plugin/store/AbstractServiceStore.java | 6 ++-
.../plugin/store/EmbeddedServiceDefsUtil.java | 46 +++++++++++---------
.../service-defs/ranger-servicedef-yarn.json | 12 +----
.../org/apache/ranger/common/AppConstants.java | 18 ++++----
.../org/apache/ranger/common/ServiceUtil.java | 4 +-
5 files changed, 43 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4915b823/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
----------------------------------------------------------------------
diff --cc agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
index d394022,0000000..86dfd1f
mode 100644,000000..100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/AbstractServiceStore.java
@@@ -1,430 -1,0 +1,434 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.store;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.util.SearchFilter;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+public abstract class AbstractServiceStore implements ServiceStore {
+ private static final Log LOG = LogFactory.getLog(AbstractServiceStore.class);
+
+
+ private static final int MAX_ACCESS_TYPES_IN_SERVICE_DEF = 1000;
+
+
+ @Override
+ public void updateTagServiceDefForAccessTypes() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceDefDBStore.updateTagServiceDefForAccessTypes()");
+ }
+ List<RangerServiceDef> allServiceDefs = getServiceDefs(new SearchFilter());
+ for (RangerServiceDef serviceDef : allServiceDefs) {
+ updateTagServiceDefForUpdatingAccessTypes(serviceDef);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceDefDBStore.updateTagServiceDefForAccessTypes()");
+ }
+ }
+
+ @Override
+ public void deleteServiceDef(Long id, Boolean forceDelete) throws Exception {
+ deleteServiceDef(id); // Ignore forceDelete flag
+ }
+
+ @Override
+ public PList<RangerServiceDef> getPaginatedServiceDefs(SearchFilter filter) throws Exception {
+ List<RangerServiceDef> resultList = getServiceDefs(filter);
+
+ return new PList<RangerServiceDef>(resultList, 0, resultList.size(),
+ (long)resultList.size(), resultList.size(), filter.getSortType(), filter.getSortBy());
+ }
+
+ @Override
+ public PList<RangerService> getPaginatedServices(SearchFilter filter) throws Exception {
+ List<RangerService> resultList = getServices(filter);
+
+ return new PList<RangerService>(resultList, 0, resultList.size(), (long)resultList.size(),
+ resultList.size(), filter.getSortType(), filter.getSortBy());
+ }
+
+ @Override
+ public PList<RangerPolicy> getPaginatedPolicies(SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getPolicies(filter);
+
+ return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(),
+ resultList.size(), filter.getSortType(), filter.getSortBy());
+ }
+
+ @Override
+ public PList<RangerPolicy> getPaginatedServicePolicies(Long serviceId, SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getServicePolicies(serviceId, filter);
+
+ return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(),
+ resultList.size(), filter.getSortType(), filter.getSortBy());
+ }
+
+ @Override
+ public PList<RangerPolicy> getPaginatedServicePolicies(String serviceName, SearchFilter filter) throws Exception {
+ List<RangerPolicy> resultList = getServicePolicies(serviceName, filter);
+
+ return new PList<RangerPolicy>(resultList, 0, resultList.size(), (long)resultList.size(),
+ resultList.size(), filter.getSortType(), filter.getSortBy());
+ }
+
+ @Override
+ public RangerPolicy getPolicyFromEventTime(String eventTimeStr, Long policyId) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret;
+ }
+
+ @Override
+ public RangerPolicy getPolicyForVersionNumber(Long policyId, Integer versionNo) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret;
+ }
+
+ @Override
+ public String getPolicyForVersionNumber(Long policyId) {
+ RangerPolicy ret = null;
+ try {
+ ret = getPolicy(policyId);
+ } catch (Exception e) {
+ // Do nothing
+ }
+ return ret == null ? null : ret.getName();
+ }
+
+ protected void preCreate(RangerBaseModelObject obj) throws Exception {
+ obj.setId(0L);
+ obj.setGuid(UUID.randomUUID().toString());
+ obj.setCreateTime(new Date());
+ obj.setUpdateTime(obj.getCreateTime());
+ obj.setVersion(1L);
+ }
+
+ protected void preCreate(RangerService service) throws Exception {
+ preCreate((RangerBaseModelObject)service);
+
+ service.setPolicyVersion(0L);
+ service.setPolicyUpdateTime(service.getCreateTime());
+ }
+
+ protected void postCreate(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ updateTagServiceDefForUpdatingAccessTypes((RangerServiceDef)obj);
+ }
+ }
+
+ protected void preUpdate(RangerBaseModelObject obj) throws Exception {
+ if(obj.getId() == null) {
+ obj.setId(0L);
+ }
+
+ if(obj.getGuid() == null) {
+ obj.setGuid(UUID.randomUUID().toString());
+ }
+
+ if(obj.getCreateTime() == null) {
+ obj.setCreateTime(new Date());
+ }
+
+ Long version = obj.getVersion();
+
+ if(version == null) {
+ version = 1L;
+ } else {
+ version = version + 1;
+ }
+
+ obj.setVersion(version);
+ obj.setUpdateTime(new Date());
+ }
+
+ protected void postUpdate(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ RangerServiceDef serviceDef = (RangerServiceDef)obj;
+
+ updateTagServiceDefForUpdatingAccessTypes(serviceDef);
+ updateServicesForServiceDefUpdate(serviceDef);
+ }
+ }
+
+ protected void preDelete(RangerBaseModelObject obj) throws Exception {
+ // TODO:
+ }
+
+ protected void postDelete(RangerBaseModelObject obj) throws Exception {
+ if(obj instanceof RangerServiceDef) {
+ updateTagServiceDefForDeletingAccessTypes(((RangerServiceDef) obj).getName());
+ }
+ }
+
+ protected long getMaxId(List<? extends RangerBaseModelObject> objs) {
+ long ret = -1;
+
+ if (objs != null) {
+ for (RangerBaseModelObject obj : objs) {
+ if (obj.getId() > ret) {
+ ret = obj.getId();
+ }
+ }
+ }
+ return ret;
+ }
+
+ // when a service-def is updated, the updated service-def should be made available to plugins
+ // this is achieved by incrementing policyVersion of all its services
+ protected void updateServicesForServiceDefUpdate(RangerServiceDef serviceDef) throws Exception {
+ boolean isTagServiceDef = StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME);
+
+ SearchFilter filter = new SearchFilter();
+
+ filter.setParam(SearchFilter.SERVICE_TYPE, serviceDef.getName());
+
+ List<RangerService> services = getServices(filter);
+
+ if(CollectionUtils.isNotEmpty(services)) {
+ for(RangerService service : services) {
+ service.setPolicyVersion(getNextVersion(service.getPolicyVersion()));
+ service.setPolicyUpdateTime(serviceDef.getUpdateTime());
+
+ updateService(service);
+
+ if(isTagServiceDef) {
+ filter = new SearchFilter();
+
+ filter.setParam(SearchFilter.TAG_SERVICE_NAME, service.getName());
+
+ List<RangerService> referrringServices = getServices(filter);
+
+ if(CollectionUtils.isNotEmpty(referrringServices)) {
+ for(RangerService referringService : referrringServices) {
+ referringService.setPolicyVersion(getNextVersion(referringService.getPolicyVersion()));
+ referringService.setPolicyUpdateTime(serviceDef.getUpdateTime());
+
+ updateService(referringService);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private RangerServiceDef.RangerAccessTypeDef findAccessTypeDef(long itemId, List<RangerServiceDef.RangerAccessTypeDef> accessTypeDefs) {
+ RangerServiceDef.RangerAccessTypeDef ret = null;
+
+ for(RangerServiceDef.RangerAccessTypeDef accessTypeDef : accessTypeDefs) {
+ if(itemId == accessTypeDef.getItemId()) {
+ ret = accessTypeDef;
+ break;
+ }
+ }
+ return ret;
+ }
+
+ private void updateTagServiceDefForUpdatingAccessTypes(RangerServiceDef serviceDef) throws Exception {
+ if (StringUtils.equals(serviceDef.getName(), EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ return;
+ }
+
++ if(EmbeddedServiceDefsUtil.instance().getTagServiceDefId() == -1) {
++ LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): tag service-def does not exist");
++ }
++
+ String serviceDefName = serviceDef.getName();
+
- RangerServiceDef tagServiceDef;
++ RangerServiceDef tagServiceDef = null;
+ try {
+ tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId());
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes" + serviceDef.getName() + "): could not find TAG ServiceDef.. ", e);
+ throw e;
+ }
+
+ if(tagServiceDef == null) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes(" + serviceDef.getName() + "): could not fund TAG ServiceDef.. ");
+
+ return;
+ }
+
+ List<RangerServiceDef.RangerAccessTypeDef> toAdd = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+ List<RangerServiceDef.RangerAccessTypeDef> toUpdate = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+ List<RangerServiceDef.RangerAccessTypeDef> toDelete = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ List<RangerServiceDef.RangerAccessTypeDef> svcDefAccessTypes = serviceDef.getAccessTypes();
+ List<RangerServiceDef.RangerAccessTypeDef> tagDefAccessTypes = tagServiceDef.getAccessTypes();
+
+ long itemIdOffset = serviceDef.getId() * (MAX_ACCESS_TYPES_IN_SERVICE_DEF + 1);
+
+ for (RangerServiceDef.RangerAccessTypeDef svcAccessType : svcDefAccessTypes) {
+ long tagAccessTypeItemId = svcAccessType.getItemId() + itemIdOffset;
+
+ RangerServiceDef.RangerAccessTypeDef tagAccessType = findAccessTypeDef(tagAccessTypeItemId, tagDefAccessTypes);
+
+ if(tagAccessType == null) {
+ tagAccessType = new RangerServiceDef.RangerAccessTypeDef();
+
+ tagAccessType.setItemId(tagAccessTypeItemId);
+ tagAccessType.setName(serviceDefName + ":" + svcAccessType.getName());
+ tagAccessType.setLabel(svcAccessType.getLabel());
+ tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel());
+
+ tagAccessType.setImpliedGrants(new HashSet<String>());
+ if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) {
+ for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) {
+ tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant);
+ }
+ }
+
+ toAdd.add(tagAccessType);
+ }
+ }
+
+ for (RangerServiceDef.RangerAccessTypeDef tagAccessType : tagDefAccessTypes) {
+ if (tagAccessType.getName().startsWith(serviceDefName + ":")) {
+ long svcAccessTypeItemId = tagAccessType.getItemId() - itemIdOffset;
+
+ RangerServiceDef.RangerAccessTypeDef svcAccessType = findAccessTypeDef(svcAccessTypeItemId, svcDefAccessTypes);
+
+ if(svcAccessType == null) { // accessType has been deleted in service
+ toDelete.add(tagAccessType);
+ continue;
+ }
+
+ boolean isUpdated = false;
+
+ if(! Objects.equals(tagAccessType.getName().substring(serviceDefName.length() + 1), svcAccessType.getName())) {
+ isUpdated = true;
+ } else if(! Objects.equals(tagAccessType.getLabel(), svcAccessType.getLabel())) {
+ isUpdated = true;
+ } else if(! Objects.equals(tagAccessType.getRbKeyLabel(), svcAccessType.getRbKeyLabel())) {
+ isUpdated = true;
+ } else {
+ Collection<String> tagImpliedGrants = tagAccessType.getImpliedGrants();
+ Collection<String> svcImpliedGrants = svcAccessType.getImpliedGrants();
+
+ int tagImpliedGrantsLen = tagImpliedGrants == null ? 0 : tagImpliedGrants.size();
+ int svcImpliedGrantsLen = svcImpliedGrants == null ? 0 : svcImpliedGrants.size();
+
+ if(tagImpliedGrantsLen != svcImpliedGrantsLen) {
+ isUpdated = true;
+ } else if(tagImpliedGrantsLen > 0) {
+ for(String svcImpliedGrant : svcImpliedGrants) {
+ if(! tagImpliedGrants.contains(serviceDefName + ":" + svcImpliedGrant)) {
+ isUpdated = true;
+ break;
+ }
+ }
+ }
+ }
+
+ if(isUpdated) {
+ tagAccessType.setName(serviceDefName + ":" + svcAccessType.getName());
+ tagAccessType.setLabel(svcAccessType.getLabel());
+ tagAccessType.setRbKeyLabel(svcAccessType.getRbKeyLabel());
+
+ tagAccessType.setImpliedGrants(new HashSet<String>());
+ if(CollectionUtils.isNotEmpty(svcAccessType.getImpliedGrants())) {
+ for(String svcImpliedGrant : svcAccessType.getImpliedGrants()) {
+ tagAccessType.getImpliedGrants().add(serviceDefName + ":" + svcImpliedGrant);
+ }
+ }
+
+ toUpdate.add(tagAccessType);
+ }
+ }
+ }
+
+ if(CollectionUtils.isNotEmpty(toAdd) || CollectionUtils.isNotEmpty(toUpdate) || CollectionUtils.isNotEmpty(toDelete)) {
+ tagDefAccessTypes.addAll(toAdd);
+ tagDefAccessTypes.removeAll(toDelete);
+
+ try {
+ updateServiceDef(tagServiceDef);
+ LOG.info("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- updated TAG service def with " + serviceDefName + " access types");
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForUpdatingAccessTypes -- Failed to update TAG ServiceDef.. ", e);
+ throw e;
+ }
+ }
+ }
+
+ private void updateTagServiceDefForDeletingAccessTypes(String serviceDefName) throws Exception {
+ if (serviceDefName.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
+ return;
+ }
+
+ RangerServiceDef tagServiceDef;
+ try {
+ tagServiceDef = this.getServiceDef(EmbeddedServiceDefsUtil.instance().getTagServiceDefId());
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not fund TAG ServiceDef.. ", e);
+ throw e;
+ }
+
+ if(tagServiceDef == null) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes(" + serviceDefName + "): could not fund TAG ServiceDef.. ");
+
+ return;
+ }
+
+ List<RangerServiceDef.RangerAccessTypeDef> accessTypes = new ArrayList<RangerServiceDef.RangerAccessTypeDef>();
+
+ for (RangerServiceDef.RangerAccessTypeDef accessType : tagServiceDef.getAccessTypes()) {
+ if (accessType.getName().startsWith(serviceDefName + ":")) {
+ accessTypes.add(accessType);
+ }
+ }
+
+ tagServiceDef.getAccessTypes().removeAll(accessTypes);
+ try {
+ updateServiceDef(tagServiceDef);
+ LOG.info("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- updated TAG service def with " + serviceDefName + " access types");
+ } catch (Exception e) {
+ LOG.error("AbstractServiceStore.updateTagServiceDefForDeletingAccessTypes -- Failed to update TAG ServiceDef.. ", e);
+ throw e;
+ }
+ }
+
+ protected long getNextVersion(Long currentVersion) {
+ return currentVersion == null ? 1L : currentVersion + 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/4915b823/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
----------------------------------------------------------------------
diff --cc agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
index 73e880b,77e2c80..959e3f4
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java
@@@ -75,10 -74,9 +75,11 @@@ public class EmbeddedServiceDefsUtil
private RangerServiceDef knoxServiceDef = null;
private RangerServiceDef stormServiceDef = null;
private RangerServiceDef yarnServiceDef = null;
- private RangerServiceDef kmsServiceDef = null;
- private RangerServiceDef kafkaServiceDef = null;
+ private RangerServiceDef kafkaServiceDef = null;
private RangerServiceDef solrServiceDef = null;
++ private RangerServiceDef tagServiceDef = null;
++
private Gson gsonBuilder = null;
@@@ -105,12 -107,8 +110,13 @@@
knoxServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KNOX_NAME);
stormServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_STORM_NAME);
yarnServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_YARN_NAME);
- kmsServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KMS_NAME);
- kafkaServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KAFKA_NAME);
+ kafkaServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_KAFKA_NAME);
solrServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SOLR_NAME);
+
++ tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME);
++
+ // Ensure that tag service def is updated with access types of all service defs
+ store.updateTagServiceDefForAccessTypes();
} catch(Throwable excp) {
LOG.fatal("EmbeddedServiceDefsUtil.init(): failed", excp);
}
@@@ -156,6 -152,6 +160,8 @@@
return getId(solrServiceDef);
}
++ public long getTagServiceDefId() { return getId(tagServiceDef); }
++
private long getId(RangerServiceDef serviceDef) {
return serviceDef == null || serviceDef.getId() == null ? -1 : serviceDef.getId().longValue();
}