You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2015/03/25 12:28:03 UTC
incubator-ranger git commit: RANGER-300 : Provide patch for migrating
repo and policies from old schema to new schema
Repository: incubator-ranger
Updated Branches:
refs/heads/master cb4eb54b6 -> 81783f322
RANGER-300 : Provide patch for migrating repo and policies from old schema to new schema
Signed-off-by: Velmurugan Periasamy <ve...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/81783f32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/81783f32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/81783f32
Branch: refs/heads/master
Commit: 81783f322bb8bf5f61aaba562fda6a0625ed67a0
Parents: cb4eb54
Author: Gautam Borad <gb...@gmail.com>
Authored: Tue Mar 24 18:08:33 2015 +0530
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Wed Mar 25 07:27:43 2015 -0400
----------------------------------------------------------------------
security-admin/scripts/setup.sh | 2 +-
.../org/apache/ranger/biz/ServiceDBStore.java | 8 +-
.../java/org/apache/ranger/db/XXPolicyDao.java | 9 +-
.../ranger/patch/PatchMigration_J10002.java | 414 +++++++++++++++++++
.../resources/META-INF/jpa_named_queries.xml | 4 +-
5 files changed, 426 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/81783f32/security-admin/scripts/setup.sh
----------------------------------------------------------------------
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index 4b0acbc..962515c 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -1363,7 +1363,7 @@ execute_java_patches(){
if [ "${c}" != "${version}" ]
then
log "[I] patch ${javaPatch} is being applied..";
- msg=`$JAVA_HOME/bin/java -cp "$app_home/WEB-INF/classes/conf:$app_home/WEB-INF/classes/lib/*:$app_home/WEB-INF/:$app_home/META-INF/:$app_home/WEB-INF/lib/*:$app_home/WEB-INF/classes/:$app_home/WEB-INF/classes/META-INF/" org.apache.ranger.patch.${className}`
+ msg=`$JAVA_HOME/bin/java -cp "$app_home/WEB-INF/classes/conf:$app_home/WEB-INF/classes/lib/*:$app_home/WEB-INF/:$app_home/META-INF/:$app_home/WEB-INF/lib/*:$app_home/WEB-INF/classes/:$app_home/WEB-INF/classes/META-INF:$SQL_CONNECTOR_JAR" org.apache.ranger.patch.${className}`
check_ret_status $? "Unable to apply patch:$javaPatch. $msg"
touch ${tempFile}
echo >> ${tempFile}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/81783f32/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 15530bf..e6513b3 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -421,7 +421,7 @@ public class ServiceDBStore implements ServiceStore {
boolean createDefaultPolicy = true;
UserSessionBase usb = ContextUtil.getCurrentUserSession();
- if (usb != null && usb.isUserAdmin()) {
+ if (usb != null && usb.isUserAdmin() || populateExistingBaseFields) {
Map<String, String> configs = service.getConfigs();
Map<String, String> validConfigs = validateRequiredConfigParams(
service, configs);
@@ -485,7 +485,7 @@ public class ServiceDBStore implements ServiceStore {
return createdService;
} else {
- LOG.debug("User id : " + usb.getUserId() + " doesn't have admin access to create repository.");
+ LOG.debug("Logged in user doesn't have admin access to create repository.");
throw restErrorUtil.createRESTException(
"Sorry, you don't have permission to perform the operation",
MessageEnums.OPER_NOT_ALLOWED_FOR_ENTITY);
@@ -678,7 +678,7 @@ public class ServiceDBStore implements ServiceStore {
throw new Exception("service-def does not exist - name=" + service.getType());
}
- XXPolicy existing = daoMgr.getXXPolicy().findByName(policy.getName());
+ XXPolicy existing = daoMgr.getXXPolicy().findByNameAndServiceId(policy.getName(), service.getId());
if(existing != null) {
throw new Exception("policy already exists: ServiceName=" + policy.getService() + "; PolicyName=" + policy.getName() + ". ID=" + existing.getId());
@@ -743,7 +743,7 @@ public class ServiceDBStore implements ServiceStore {
boolean renamed = !StringUtils.equalsIgnoreCase(policy.getName(), existing.getName());
if(renamed) {
- XXPolicy newNamePolicy = daoMgr.getXXPolicy().findByName(policy.getName());
+ XXPolicy newNamePolicy = daoMgr.getXXPolicy().findByNameAndServiceId(policy.getName(), service.getId());
if(newNamePolicy != null) {
throw new Exception("another policy already exists with name '" + policy.getName() + "'. ID=" + newNamePolicy.getId());
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/81783f32/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
index e2b4fcf..89eff56 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyDao.java
@@ -19,14 +19,15 @@ public class XXPolicyDao extends BaseDao<XXPolicy> {
super(daoManager);
}
- public XXPolicy findByName(String polName) {
- if (polName == null) {
+ public XXPolicy findByNameAndServiceId(String polName, Long serviceId) {
+ if (polName == null || serviceId == null) {
return null;
}
try {
XXPolicy xPol = getEntityManager()
- .createNamedQuery("XXPolicy.findByName", tClass)
- .setParameter("polName", polName).getSingleResult();
+ .createNamedQuery("XXPolicy.findByNameAndServiceId", tClass)
+ .setParameter("polName", polName).setParameter("serviceId", serviceId)
+ .getSingleResult();
return xPol;
} catch (NoResultException e) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/81783f32/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java b/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java
new file mode 100644
index 0000000..34b6541
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/patch/PatchMigration_J10002.java
@@ -0,0 +1,414 @@
+package org.apache.ranger.patch;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.biz.ServiceDBStore;
+import org.apache.ranger.common.AppConstants;
+import org.apache.ranger.common.JSONUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.StringUtil;
+import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.entity.XXAsset;
+import org.apache.ranger.entity.XXAuditMap;
+import org.apache.ranger.entity.XXPolicy;
+import org.apache.ranger.entity.XXPolicyConditionDef;
+import org.apache.ranger.entity.XXPortalUser;
+import org.apache.ranger.entity.XXResource;
+import org.apache.ranger.entity.XXServiceConfigDef;
+import org.apache.ranger.entity.XXServiceDef;
+import org.apache.ranger.patch.BaseLoader;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItem;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemAccess;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyItemCondition;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil;
+import org.apache.ranger.service.RangerPolicyService;
+import org.apache.ranger.service.XPermMapService;
+import org.apache.ranger.service.XPolicyService;
+import org.apache.ranger.util.CLIUtil;
+import org.apache.ranger.view.VXPermMap;
+import org.apache.ranger.view.VXPermObj;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class PatchMigration_J10002 extends BaseLoader {
+ private static Logger logger = Logger.getLogger(PatchMigration_J10002.class);
+
+ @Autowired
+ RangerDaoManager daoMgr;
+
+ @Autowired
+ ServiceDBStore svcDBStore;
+
+ @Autowired
+ JSONUtil jsonUtil;
+
+ @Autowired
+ RangerPolicyService policyService;
+
+ @Autowired
+ StringUtil stringUtil;
+
+ @Autowired
+ XPolicyService xPolService;
+
+ @Autowired
+ XPermMapService xPermMapService;
+
+ private static int policyCounter = 0;
+ private static int serviceCounter = 0;
+
+ public static void main(String[] args) {
+ logger.info("main()");
+ try {
+ PatchMigration_J10002 loader = (PatchMigration_J10002) CLIUtil.getBean(PatchMigration_J10002.class);
+ loader.init();
+ while (loader.isMoreToProcess()) {
+ loader.load();
+ }
+ logger.info("Load complete. Exiting!!!");
+ System.exit(0);
+ } catch (Exception e) {
+ logger.error("Error loading", e);
+ System.exit(1);
+ }
+ }
+
+ @Override
+ public void init() throws Exception {
+ // Do Nothing
+ }
+
+ @Override
+ public void execLoad() {
+ logger.info("==> MigrationPatch.execLoad()");
+ try {
+ migrateServicesToNewSchema();
+ migratePoliciesToNewSchema();
+ } catch (Exception e) {
+ logger.error("Error whille migrating data.", e);
+ }
+ logger.info("<== MigrationPatch.execLoad()");
+ }
+
+ @Override
+ public void printStats() {
+ logger.info("Total Number of migrated repositories/services: " + serviceCounter);
+ logger.info("Total Number of migrated resources/policies: " + policyCounter);
+ }
+
+ public void migrateServicesToNewSchema() throws Exception {
+ logger.info("==> MigrationPatch.migrateServicesToNewSchema()");
+
+ try {
+ List<XXAsset> repoList = daoMgr.getXXAsset().getAll();
+
+ if (repoList.size() <= 0) {
+ return;
+ }
+ if (repoList.size() > 0) {
+ EmbeddedServiceDefsUtil.instance().init(svcDBStore);
+ }
+
+ svcDBStore.setPopulateExistingBaseFields(true);
+ for (XXAsset xAsset : repoList) {
+
+ if (xAsset.getActiveStatus() == AppConstants.STATUS_DELETED) {
+ continue;
+ }
+
+ RangerService existing = svcDBStore.getServiceByName(xAsset.getName());
+ if (existing != null) {
+ logger.info("Repository/Service already exists. Ignoring migration of repo: " + xAsset.getName());
+ continue;
+ }
+
+ RangerService service = new RangerService();
+ service = mapXAssetToService(service, xAsset);
+
+ service = svcDBStore.createService(service);
+
+ serviceCounter++;
+ logger.info("New Service created. ServiceName: " + service.getName());
+ }
+ svcDBStore.setPopulateExistingBaseFields(false);
+ } catch (Exception e) {
+ throw new Exception("Error while migrating data to new Plugin Schema.", e);
+ }
+ logger.info("<== MigrationPatch.migrateServicesToNewSchema()");
+ }
+
+ public void migratePoliciesToNewSchema() throws Exception {
+ logger.info("==> MigrationPatch.migratePoliciesToNewSchema()");
+
+ try {
+ List<XXResource> resList = daoMgr.getXXResource().getAll();
+ if (resList.size() <= 0) {
+ return;
+ }
+
+ svcDBStore.setPopulateExistingBaseFields(true);
+ for (XXResource xRes : resList) {
+
+ if (xRes.getResourceStatus() == AppConstants.STATUS_DELETED) {
+ continue;
+ }
+
+ XXAsset xAsset = daoMgr.getXXAsset().getById(xRes.getAssetId());
+ if (xAsset == null) {
+ logger.error("No Repository found for policyName: " + xRes.getPolicyName());
+ continue;
+ }
+
+ RangerService service = svcDBStore.getServiceByName(xAsset.getName());
+
+ if (service == null) {
+ logger.error("No Service found for policy. Ignoring migration of such policy, policyName: "
+ + xRes.getPolicyName());
+ continue;
+ }
+
+ XXPolicy existing = daoMgr.getXXPolicy().findByNameAndServiceId(xRes.getPolicyName(), service.getId());
+ if (existing != null) {
+ logger.info("Policy already exists. Ignoring migration of policy: " + existing.getName());
+ continue;
+ }
+
+ RangerPolicy policy = new RangerPolicy();
+ policy = mapXResourceToPolicy(policy, xRes, service);
+
+ policy = svcDBStore.createPolicy(policy);
+
+ policyCounter++;
+ logger.info("New policy created. policyName: " + policy.getName());
+ }
+ svcDBStore.setPopulateExistingBaseFields(false);
+ } catch (Exception e) {
+ throw new Exception("Error while migrating data to new Plugin Schema.", e);
+ }
+ logger.info("<== MigrationPatch.migratePoliciesToNewSchema()");
+ }
+
+ private RangerService mapXAssetToService(RangerService service, XXAsset xAsset) throws Exception {
+
+ String type = "";
+ String name = xAsset.getName();
+ String description = xAsset.getDescription();
+ Map<String, String> configs = null;
+
+ int typeInt = xAsset.getAssetType();
+ XXServiceDef serviceDef = daoMgr.getXXServiceDef().findByName(AppConstants.getLabelFor_AssetType(typeInt).toLowerCase());
+
+ if (serviceDef == null) {
+ throw new Exception("No ServiceDefinition found for repository: " + name);
+ }
+ type = serviceDef.getName();
+ configs = jsonUtil.jsonToMap(xAsset.getConfig());
+
+ List<XXServiceConfigDef> mandatoryConfigs = daoMgr.getXXServiceConfigDef().findByServiceDefName(type);
+ for (XXServiceConfigDef serviceConf : mandatoryConfigs) {
+ if (serviceConf.getIsMandatory()) {
+ if (!stringUtil.isEmpty(configs.get(serviceConf.getName()))) {
+ continue;
+ }
+ String dataType = serviceConf.getType();
+ String defaultValue = serviceConf.getDefaultvalue();
+
+ if (stringUtil.isEmpty(defaultValue)) {
+ defaultValue = getDefaultValueForDataType(dataType);
+ }
+ configs.put(serviceConf.getName(), defaultValue);
+ }
+ }
+
+ service.setType(type);
+ service.setName(name);
+ service.setDescription(description);
+ service.setConfigs(configs);
+
+ service.setCreateTime(xAsset.getCreateTime());
+ service.setUpdateTime(xAsset.getUpdateTime());
+
+ XXPortalUser createdByUser = daoMgr.getXXPortalUser().getById(xAsset.getAddedByUserId());
+ XXPortalUser updByUser = daoMgr.getXXPortalUser().getById(xAsset.getUpdatedByUserId());
+
+ if (createdByUser != null) {
+ service.setCreatedBy(createdByUser.getLoginId());
+ }
+ if (updByUser != null) {
+ service.setUpdatedBy(updByUser.getLoginId());
+ }
+ service.setId(xAsset.getId());
+
+ return service;
+ }
+
+ private String getDefaultValueForDataType(String dataType) {
+
+ String defaultValue = "";
+ switch (dataType) {
+ case "int":
+ defaultValue = "0";
+ break;
+ case "string":
+ defaultValue = "unknown";
+ break;
+ case "bool":
+ defaultValue = "false";
+ break;
+ case "enum":
+ defaultValue = "0";
+ break;
+ case "password":
+ defaultValue = "password";
+ break;
+ default:
+ break;
+ }
+ return defaultValue;
+ }
+
+ private RangerPolicy mapXResourceToPolicy(RangerPolicy policy, XXResource xRes, RangerService service) {
+
+ String serviceName = service.getName();
+ String serviceDef = service.getType();
+ String name = xRes.getPolicyName();
+ String description = xRes.getDescription();
+ Boolean isAuditEnabled = true;
+ Boolean isEnabled = true;
+ Map<String, RangerPolicyResource> resources = new HashMap<String, RangerPolicyResource>();
+ List<RangerPolicyItem> policyItems = new ArrayList<RangerPolicyItem>();
+
+ List<XXAuditMap> auditMapList = daoMgr.getXXAuditMap().findByResourceId(xRes.getId());
+ if (stringUtil.isEmpty(auditMapList)) {
+ isAuditEnabled = false;
+ }
+ if (xRes.getResourceStatus() == AppConstants.STATUS_DISABLED) {
+ isEnabled = false;
+ }
+
+ boolean tableExcludes = false;
+ boolean columnExcludes = false;
+
+ if (xRes.getTableType() == AppConstants.POLICY_EXCLUSION) {
+ tableExcludes = true;
+ }
+ if (xRes.getColumnType() == AppConstants.POLICY_EXCLUSION) {
+ columnExcludes = true;
+ }
+
+ if (serviceDef.equalsIgnoreCase("hdfs")) {
+ resources.put("path", new RangerPolicyResource(Arrays.asList(xRes.getName()), false, AppConstants
+ .getBooleanFor_BooleanValue(xRes.getIsRecursive())));
+
+ } else if (serviceDef.equalsIgnoreCase("hbase")) {
+ resources.put("table", new RangerPolicyResource(Arrays.asList(xRes.getTables()), tableExcludes, false));
+ resources.put("column", new RangerPolicyResource(Arrays.asList(xRes.getColumns()), columnExcludes, false));
+ resources.put("column-family", new RangerPolicyResource(Arrays.asList(xRes.getColumnFamilies()), false, false));
+
+ } else if (serviceDef.equalsIgnoreCase("hive")) {
+ resources.put("table", new RangerPolicyResource(Arrays.asList(xRes.getTables()), tableExcludes, false));
+ resources.put("column", new RangerPolicyResource(Arrays.asList(xRes.getColumns()), columnExcludes, false));
+ resources.put("database", new RangerPolicyResource(Arrays.asList(xRes.getDatabases()), false, false));
+ resources.put("udf", new RangerPolicyResource(Arrays.asList(xRes.getUdfs()), false, false));
+ } else if (serviceDef.equalsIgnoreCase("knox")) {
+ resources.put("topology", new RangerPolicyResource(Arrays.asList(xRes.getTopologies()), false, false));
+ resources.put("service", new RangerPolicyResource(Arrays.asList(xRes.getServices()), false, false));
+ } else if (serviceDef.equalsIgnoreCase("storm")) {
+ resources.put("topology", new RangerPolicyResource(Arrays.asList(xRes.getTopologies()), false, false));
+ }
+
+ policyItems = getPolicyItemListForRes(xRes, serviceDef);
+
+ policy.setService(serviceName);
+ policy.setName(name);
+ policy.setDescription(description);
+ policy.setIsAuditEnabled(isAuditEnabled);
+ policy.setIsEnabled(isEnabled);
+ policy.setResources(resources);
+ policy.setPolicyItems(policyItems);
+
+ policy.setCreateTime(xRes.getCreateTime());
+ policy.setUpdateTime(xRes.getUpdateTime());
+
+ XXPortalUser createdByUser = daoMgr.getXXPortalUser().getById(xRes.getAddedByUserId());
+ XXPortalUser updByUser = daoMgr.getXXPortalUser().getById(xRes.getUpdatedByUserId());
+
+ if (createdByUser != null) {
+ policy.setCreatedBy(createdByUser.getLoginId());
+ }
+ if (updByUser != null) {
+ policy.setUpdatedBy(updByUser.getLoginId());
+ }
+
+ policy.setId(xRes.getId());
+
+ return policy;
+ }
+
+ private List<RangerPolicyItem> getPolicyItemListForRes(XXResource xRes, String serviceDefName) {
+ List<RangerPolicyItem> policyItems = new ArrayList<RangerPolicyItem>();
+
+ SearchCriteria sc = new SearchCriteria();
+ sc.addParam("resourceId", xRes.getId());
+ List<VXPermMap> permMapList = xPermMapService.searchXPermMaps(sc).getVXPermMaps();
+ List<VXPermObj> permObjList = xPolService.mapPermMapToPermObj(permMapList);
+
+ XXServiceDef svcDef = daoMgr.getXXServiceDef().findByName(serviceDefName);
+ if (svcDef == null) {
+ return new ArrayList<RangerPolicyItem>();
+ }
+
+ XXPolicyConditionDef policyCond = daoMgr.getXXPolicyConditionDef().findByServiceDefIdAndName(svcDef.getId(),
+ "ip-range");
+
+ for (VXPermObj permObj : permObjList) {
+
+ List<String> permList = permObj.getPermList();
+ if (permList == null) {
+ continue;
+ }
+
+ RangerPolicyItem policyItem = new RangerPolicyItem();
+ List<RangerPolicyItemAccess> accesses = new ArrayList<RangerPolicyItemAccess>();
+ List<RangerPolicyItemCondition> conditions = new ArrayList<RangerPolicyItemCondition>();
+
+ if (permObj.getPermList().contains("Admin")) {
+ policyItem.setDelegateAdmin(true);
+ }
+
+ for (String perm : permList) {
+ RangerPolicyItemAccess access = new RangerPolicyItemAccess();
+ access.setIsAllowed(true);
+ access.setType(perm);
+ accesses.add(access);
+ }
+ if (!stringUtil.isEmpty(permObj.getIpAddress()) && policyCond != null) {
+ RangerPolicyItemCondition condition = new RangerPolicyItemCondition();
+ condition.setType("ip-range");
+
+ List<String> ipRangeList = Arrays.asList(permObj.getIpAddress());
+
+ condition.setValues(ipRangeList);
+ conditions.add(condition);
+ }
+
+ policyItem.setUsers(permObj.getUserList());
+ policyItem.setGroups(permObj.getGroupList());
+ policyItem.setAccesses(accesses);
+ policyItem.setConditions(conditions);
+
+ policyItems.add(policyItem);
+ }
+ return policyItems;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/81783f32/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index e304fd4..7a0fe30 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -200,8 +200,8 @@
</named-query>
<!-- XXPolicy -->
- <named-query name="XXPolicy.findByName">
- <query>select obj from XXPolicy obj where obj.name = :polName</query>
+ <named-query name="XXPolicy.findByNameAndServiceId">
+ <query>select obj from XXPolicy obj where obj.name = :polName and obj.service = :serviceId</query>
</named-query>
<named-query name="XXPolicy.findByServiceId">