You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ga...@apache.org on 2016/03/05 15:39:52 UTC
incubator-ranger git commit: RANGER-271 : Audit migration script from
DB to Solr
Repository: incubator-ranger
Updated Branches:
refs/heads/master f46c5cd91 -> 6dbd2b8f1
RANGER-271 : Audit migration script from DB to Solr
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/6dbd2b8f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/6dbd2b8f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/6dbd2b8f
Branch: refs/heads/master
Commit: 6dbd2b8f14ded9f1cbcfe896b1d1c2b2141ce41d
Parents: f46c5cd
Author: Pradeep Agrawal <pr...@freestoneinfotech.com>
Authored: Sat Mar 5 17:30:27 2016 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Sat Mar 5 20:09:41 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/ranger/common/DateUtil.java | 9 +-
.../org/apache/ranger/db/XXAccessAuditDao.java | 34 ++-
.../patch/cliutil/DbToSolrMigrationUtil.java | 222 +++++++++++++++++++
.../resources/META-INF/jpa_named_queries.xml | 6 +
.../src/main/resources/META-INF/persistence.xml | 1 +
5 files changed, 270 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java b/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java
index b710020..92150ee 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java
@@ -105,5 +105,12 @@ public class DateUtil {
return null;
}
}
-
+ public static Date getLocalDateForUTCDate(Date date) {
+ Calendar local = Calendar.getInstance();
+ int offset = local.getTimeZone().getOffset(local.getTimeInMillis());
+ GregorianCalendar utc = new GregorianCalendar();
+ utc.setTimeInMillis(date.getTime());
+ utc.add(Calendar.MILLISECOND, offset);
+ return utc.getTime();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java
index cd1add6..432dfa9 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java
@@ -19,13 +19,45 @@
package org.apache.ranger.db;
+import java.util.ArrayList;
+import java.util.List;
+import javax.persistence.NoResultException;
+import org.apache.log4j.Logger;
import org.apache.ranger.common.db.BaseDao;
import org.apache.ranger.entity.XXAccessAudit;
public class XXAccessAuditDao extends BaseDao<XXAccessAudit> {
-
+ private static Logger logger = Logger.getLogger(XXAccessAuditDao.class);
public XXAccessAuditDao( RangerDaoManagerBase daoManager ) {
super(daoManager, "loggingPU");
}
+ public Long getMaxIdOfXXAccessAudit(){
+ Long maxXXAccessAuditID=Long.valueOf(0L);
+ try {
+ maxXXAccessAuditID = (Long) getEntityManager()
+ .createNamedQuery("XXAccessAudit.getMaxIdOfXXAccessAudit", Long.class)
+ .getSingleResult();
+ } catch (NoResultException e) {
+ logger.debug(e.getMessage());
+ }finally{
+ if(maxXXAccessAuditID==null){
+ maxXXAccessAuditID=Long.valueOf(0L);
+ }
+ }
+ return maxXXAccessAuditID;
+ }
+ public List<XXAccessAudit> getByIdRange(long idFrom,long idTo){
+ //idFrom and idTo both exclusive
+ List<XXAccessAudit> xXAccessAuditList = new ArrayList<XXAccessAudit>();
+ try {
+ xXAccessAuditList= getEntityManager().createNamedQuery("XXAccessAudit.getByIdRange", tClass)
+ .setParameter("idFrom", idFrom)
+ .setParameter("idTo", idTo)
+ .getResultList();
+ } catch (NoResultException e) {
+ logger.debug(e.getMessage());
+ }
+ return xXAccessAuditList;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
new file mode 100644
index 0000000..9ce1a12
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.patch.cliutil;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.apache.log4j.Logger;
+import org.apache.ranger.db.RangerDaoManager;
+import org.apache.ranger.entity.XXAccessAudit;
+import org.apache.ranger.patch.BaseLoader;
+import org.apache.ranger.solr.SolrAccessAuditsService;
+import org.apache.ranger.authorization.utils.StringUtil;
+import org.apache.ranger.common.DateUtil;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.util.CLIUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class DbToSolrMigrationUtil extends BaseLoader {
+ private static Logger logger = Logger.getLogger(DbToSolrMigrationUtil.class);
+ private HttpSolrServer solrServer=null;
+ private final static String CHECK_FILE_NAME = "migration_check_file.txt";
+ private final static Charset ENCODING = StandardCharsets.UTF_8;
+ @Autowired
+ RangerDaoManager daoManager;
+ @Autowired
+ SolrAccessAuditsService solrAccessAuditsService;
+
+ public static void main(String[] args) {
+ logger.info("main()");
+ try {
+ DbToSolrMigrationUtil loader = (DbToSolrMigrationUtil) CLIUtil
+ .getBean(DbToSolrMigrationUtil.class);
+
+ loader.init();
+ while (loader.isMoreToProcess()) {
+ loader.load();
+ }
+ logger.info("Load complete. Exiting!!!");
+ System.exit(0);
+ } catch (Exception e) {
+ logger.error("Error loading", e);
+ System.exit(1);
+ }
+ }
+
+ @Override
+ public void init() throws Exception {
+ String solrURL=PropertiesUtil.getProperty("ranger.audit.solr.urls");
+ logger.info("solrURL:"+solrURL);
+ solrServer = new HttpSolrServer(solrURL);
+ }
+
+ @Override
+ public void execLoad() {
+ logger.info("==> DbToSolrMigrationUtil.execLoad() Start.");
+ migrateAuditDbLogsToSolr();
+ logger.info("<== DbToSolrMigrationUtil.execLoad() End.");
+ }
+
+ public void migrateAuditDbLogsToSolr() {
+ long maxXXAccessAuditID = daoManager.getXXAccessAudit().getMaxIdOfXXAccessAudit();
+ if(maxXXAccessAuditID==0){
+ logger.info("Access Audit log does not exist.");
+ return;
+ }
+ long maxMigratedID=0;
+ try {
+ maxMigratedID = readMigrationStatusFile(CHECK_FILE_NAME);
+ } catch (IOException ex) {
+ logger.error("Failed to read migration status from file " + CHECK_FILE_NAME, ex);
+ }
+ logger.info("ID of the last available audit log: "+ maxXXAccessAuditID);
+ if(maxMigratedID > 0) {
+ logger.info("ID of the last migrated audit log: "+ maxMigratedID);
+ }
+ if(maxMigratedID>=maxXXAccessAuditID){
+ logger.info("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID);
+ return;
+ }
+ long maxRowsPerBatch=10000;
+ //To ceil the actual division result i.e noOfBatches=maxXXAccessAuditID/maxRowsPerBatch
+ long noOfBatches=((maxXXAccessAuditID-maxMigratedID)+maxRowsPerBatch-1)/maxRowsPerBatch;
+ long rangeStart=maxMigratedID;
+ long rangeEnd=maxXXAccessAuditID-maxMigratedID<=maxRowsPerBatch ? maxXXAccessAuditID : rangeStart+maxRowsPerBatch;
+ long startTimeInMS=0;
+ long timeTaken=0;
+ List<XXAccessAudit> xXAccessAuditList=null;
+ for(long index=1;index<=noOfBatches;index++){
+ logger.info("Batch "+ index+" of total "+noOfBatches);
+ startTimeInMS=System.currentTimeMillis();
+ //rangeStart and rangeEnd both exclusive, if we add +1 in maxRange
+ xXAccessAuditList=daoManager.getXXAccessAudit().getByIdRange(rangeStart,rangeEnd+1);
+ for(XXAccessAudit xXAccessAudit:xXAccessAuditList){
+ if(xXAccessAudit!=null){
+ try {
+ send2solr(xXAccessAudit);
+ } catch (Throwable e) {
+ logger.error("Error while writing audit log id '"+xXAccessAudit.getId()+"' to Solr.", e);
+ writeMigrationStatusFile(xXAccessAudit.getId(),CHECK_FILE_NAME);
+ logger.info("Stopping migration process!");
+ return;
+ }
+ }
+ }
+ timeTaken=(System.currentTimeMillis()-startTimeInMS);
+ logger.info("Batch #" + index + ": time taken:"+timeTaken+" ms");
+ if(rangeEnd<maxXXAccessAuditID){
+ writeMigrationStatusFile(rangeEnd,CHECK_FILE_NAME);
+ }else{
+ writeMigrationStatusFile(maxXXAccessAuditID,CHECK_FILE_NAME);
+ }
+ rangeStart=rangeEnd;
+ rangeEnd=rangeEnd+maxRowsPerBatch;
+ }
+
+ }
+
+ public void send2solr(XXAccessAudit xXAccessAudit) throws Throwable {
+ boolean uidIsString = true;
+ SolrInputDocument document = new SolrInputDocument();
+ document.addField("id", xXAccessAudit.getId());
+ document.addField("access", xXAccessAudit.getAccessType());
+ document.addField("enforcer", xXAccessAudit.getAclEnforcer());
+ document.addField("agent", xXAccessAudit.getAgentId());
+ document.addField("repo", xXAccessAudit.getRepoName());
+ document.addField("sess", xXAccessAudit.getSessionId());
+ document.addField("reqUser", xXAccessAudit.getRequestUser());
+ document.addField("reqData", xXAccessAudit.getRequestData());
+ document.addField("resource", xXAccessAudit.getResourcePath());
+ document.addField("cliIP", xXAccessAudit.getClientIP());
+ document.addField("logType", "RangerAudit");
+ document.addField("result", xXAccessAudit.getAccessResult());
+ document.addField("policy", xXAccessAudit.getPolicyId());
+ document.addField("repoType", xXAccessAudit.getRepoType());
+ document.addField("resType", xXAccessAudit.getResourceType());
+ document.addField("reason", xXAccessAudit.getResultReason());
+ document.addField("action", xXAccessAudit.getAction());
+ document.addField("evtTime", DateUtil.getLocalDateForUTCDate(xXAccessAudit.getEventTime()));
+ document.addField("seq_num", xXAccessAudit.getSequenceNumber());
+ document.addField("event_count", xXAccessAudit.getEventCount());
+ document.addField("event_dur_ms", xXAccessAudit.getEventDuration());
+ document.addField("tags", xXAccessAudit.getTags());
+ //If ID is not set, then we should add it.
+ SolrInputField idField = document.getField("id");
+ if( idField == null) {
+ Object uid = null;
+ if(uidIsString) {
+ uid = UUID.randomUUID().toString();
+ }
+ document.setField("id", uid);
+ }
+
+ UpdateResponse response = solrServer.add(document);
+ if (response.getStatus() != 0) {
+ logger.info("Response=" + response.toString() + ", status= "
+ + response.getStatus() + ", event=" + xXAccessAudit.toString());
+ throw new Exception("Failed to send audit event ID=" + xXAccessAudit.getId());
+ }
+ }
+
+ private Long readMigrationStatusFile(String aFileName) throws IOException {
+ Long migratedDbID=0L;
+ Path path = Paths.get(aFileName);
+ if (Files.exists(path) && Files.isRegularFile(path)) {
+ List<String> fileContents=Files.readAllLines(path, ENCODING);
+ if(fileContents!=null && fileContents.size()>=1){
+ String line=fileContents.get(fileContents.size()-1).trim();
+ if(!StringUtil.isEmpty(line)){
+ try{
+ migratedDbID=Long.parseLong(line);
+ }catch(Exception ex){
+ }
+ }
+ }
+ }
+ return migratedDbID;
+ }
+
+ private void writeMigrationStatusFile(Long DbID, String aFileName) {
+ try{
+ Path path = Paths.get(aFileName);
+ List<String> fileContents=new ArrayList<String>();
+ fileContents.add(String.valueOf(DbID));
+ Files.write(path, fileContents, ENCODING);
+ }catch(IOException ex){
+ logger.error("Failed to update migration status to file " + CHECK_FILE_NAME, ex);
+ }catch(Exception ex){
+ logger.error("Error while updating migration status to file " + CHECK_FILE_NAME, ex);
+ }
+ }
+ @Override
+ public void printStats() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index 3c84b62..be9b69e 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -923,4 +923,10 @@
<query>select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemGroupPerm polItemGroupPerm where
obj.id = polItem.policyId and polItem.id = polItemGroupPerm.policyItemId and polItemGroupPerm.groupId = :groupId</query>
</named-query>
+ <named-query name="XXAccessAudit.getMaxIdOfXXAccessAudit">
+ <query>select max(obj.id) from XXAccessAudit obj</query>
+ </named-query>
+ <named-query name="XXAccessAudit.getByIdRange">
+ <query>select obj from XXAccessAudit obj WHERE obj.id > :idFrom AND obj.id < :idTo</query>
+ </named-query>
</entity-mappings>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/persistence.xml b/security-admin/src/main/resources/META-INF/persistence.xml
index 344e2e9..e34ac0a 100644
--- a/security-admin/src/main/resources/META-INF/persistence.xml
+++ b/security-admin/src/main/resources/META-INF/persistence.xml
@@ -41,6 +41,7 @@
</properties>
</persistence-unit>
<persistence-unit name="loggingPU">
+ <mapping-file>META-INF/jpa_named_queries.xml</mapping-file>
<class>org.apache.ranger.entity.XXAccessAudit</class>
<shared-cache-mode>NONE</shared-cache-mode>