You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/22 16:16:02 UTC

[66/95] [abbrv] git commit: - Added implementation for distributing jobs file wise

- Added implementation for distributing jobs file wise

TO DO:
- test the above
- Add endpoint to get file Job information as now the current will only give overall job status


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a26716d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a26716d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a26716d2

Branch: refs/heads/import-feature
Commit: a26716d23b9f7cb281b752c28ecba0c605df6c99
Parents: cc0ad6e
Author: Pooja Jain <pj...@apigee.com>
Authored: Tue Jul 29 18:59:51 2014 -0700
Committer: Pooja Jain <pj...@apigee.com>
Committed: Tue Jul 29 18:59:51 2014 -0700

----------------------------------------------------------------------
 .../persistence/entities/FileImport.java        | 120 +++++++++
 .../management/importUG/FileImportJob.java      |  96 +++++++
 .../usergrid/management/importUG/ImportJob.java |  19 ++
 .../management/importUG/ImportService.java      |  31 +++
 .../management/importUG/ImportServiceImpl.java  | 263 ++++++++++++-------
 .../resources/usergrid-services-context.xml     |   1 +
 6 files changed, 438 insertions(+), 92 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
new file mode 100644
index 0000000..8688bec
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.entities;
+
+
+import org.apache.usergrid.persistence.TypedEntity;
+import org.apache.usergrid.persistence.annotations.EntityProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+
+/**
+ * Contains state information for an Entity Job
+ */
+@XmlRootElement
+public class FileImport extends TypedEntity {
+    //canceled , and expired states aren't used in current iteration.
+    public static enum State {
+        CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
+    }
+
+    @EntityProperty
+    protected State curState;
+
+    /**
+     * Error message
+     */
+    @EntityProperty
+    protected String errorMessage;
+
+
+    /**
+     * file name
+     */
+    @EntityProperty
+    protected String fileName;
+
+    /**
+     * File completion Status
+     */
+    @EntityProperty
+    protected Boolean completed;
+
+    /**
+     * LastUpdatedUUID
+     */
+    @EntityProperty
+    protected String lastUpdatedUUID;
+
+    public FileImport() {
+    }
+
+
+    public String getLastUpdatedUUID() {
+        return lastUpdatedUUID;
+    }
+
+
+    public void setLastUpdatedUUID( final String lastUpdatedUUID ) {
+        this.lastUpdatedUUID = lastUpdatedUUID;
+    }
+
+    public Boolean getCompleted() {
+        return completed;
+    }
+
+
+    public void setCompleted( final Boolean completed ) {
+        this.completed  = completed;
+    }
+
+    //state should moved to a derived state, but it is not there yet.
+    @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+    @EntityProperty
+    public void setState( State setter ) {
+        curState = setter;
+    }
+
+
+    @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+    @EntityProperty
+    public State getState() { return curState; }
+
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+
+    public void setErrorMessage( final String errorMessage ) {
+        this.errorMessage = errorMessage;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+
+    public void setFileName( final String fileName ) {
+        this.fileName = fileName;
+    }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
new file mode 100644
index 0000000..1e14116
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.management.importUG;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.FileImport;
+import org.apache.usergrid.persistence.entities.JobData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+
+/**
+ * Created by ApigeeCorporation on 7/8/14.
+ */
+@Component("fileImportJob")
+public class FileImportJob extends OnlyOnceJob {
+
+    public static final String FILE_IMPORT_ID = "fileImportId";
+    private static final Logger logger = LoggerFactory.getLogger(FileImportJob.class);
+
+    @Autowired
+    ImportService importService;
+
+    //injected the Entity Manager Factory
+    protected EntityManagerFactory emf;
+
+    public FileImportJob() {
+        logger.info( "FileImportJob created " + this );
+    }
+
+    @Override
+    protected void doJob(JobExecution jobExecution) throws Exception {
+        logger.info( "execute FileImportJob {}", jobExecution );
+
+        JobData jobData = jobExecution.getJobData();
+        if ( jobData == null ) {
+            logger.error( "jobData cannot be null" );
+            return;
+        }
+
+        jobExecution.heartbeat();
+        try {
+            importService.FileParser( jobExecution );
+        }
+        catch ( Exception e ) {
+            logger.error( " Import Service failed to complete file import job" );
+            logger.error(e.getMessage());
+            throw e;
+        }
+
+        logger.info( "executed FileImportJob process completed" );
+    }
+
+    @Override
+    protected long getDelay(JobExecution execution) throws Exception {
+        return 100;
+    }
+
+    @Autowired
+    public void setImportService( final ImportService importService ) {
+        this.importService = importService;
+    }
+
+    @Override
+    public void dead( final JobExecution execution ) throws Exception {
+
+        EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+        FileImport fileImport = importService.getFileImportEntity(execution);
+        fileImport.setErrorMessage("The Job has been tried maximum times but still failed");
+        fileImport.setState(FileImport.State.FAILED);
+        rootEm.update(fileImport);
+        //To change body of implemented methods use File | Settings | File Templates.
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
index db0c4c8..3b1a0f2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
@@ -19,12 +19,17 @@ package org.apache.usergrid.management.importUG;
 
 import org.apache.usergrid.batch.JobExecution;
 import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Import;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+
 /**
  * Created by ApigeeCorporation on 7/8/14.
  */
@@ -37,6 +42,9 @@ public class ImportJob extends OnlyOnceJob {
     @Autowired
     ImportService importService;
 
+    //injected the Entity Manager Factory
+    protected EntityManagerFactory emf;
+
     public ImportJob() {
         logger.info( "ImportJob created " + this );
     }
@@ -73,4 +81,15 @@ public class ImportJob extends OnlyOnceJob {
     public void setImportService( final ImportService importService ) {
         this.importService = importService;
     }
+
+    @Override
+    public void dead( final JobExecution execution ) throws Exception {
+
+        EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+        Import importUG = importService.getImportEntity(execution);
+        importUG.setErrorMessage("The Job has been tried maximum times but still failed");
+        importUG.setState(Import.State.FAILED);
+        rootEm.update(importUG);
+        //To change body of implemented methods use File | Settings | File Templates.
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
index 6866036..d0be747 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
@@ -18,6 +18,9 @@
 package org.apache.usergrid.management.importUG;
 
 import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.entities.FileImport;
+import org.apache.usergrid.persistence.entities.Import;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -35,11 +38,23 @@ public interface ImportService {
     UUID schedule(Map<String, Object> json) throws Exception;
 
     /**
+     * Schedules the import to execute
+     */
+    UUID scheduleFile(File fileName,EntityRef importRef) throws Exception;
+
+    /**
      * Perform the import from the external resource
      */
     void doImport(JobExecution jobExecution) throws Exception;
 
     /**
+     * Parses the input file and creates entities
+     * @param jobExecution
+     * @throws Exception
+     */
+    void FileParser(JobExecution jobExecution) throws Exception;
+
+    /**
      * Returns the current state of the service.
      */
     String getState(UUID state) throws Exception;
@@ -51,4 +66,20 @@ public interface ImportService {
      */
     ArrayList<File> getEphemeralFile();
 
+    /**
+     *
+     * @param jobExecution
+     * @return FileImportEntity
+     * @throws Exception
+     */
+    FileImport getFileImportEntity( final JobExecution jobExecution ) throws Exception;
+    /**
+     *
+     * @param jobExecution
+     * @return getImportEntity
+     * @throws Exception
+     */
+    Import getImportEntity( final JobExecution jobExecution ) throws Exception;
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
index 59281b4..340b058 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
@@ -22,10 +22,8 @@ import org.apache.usergrid.batch.service.SchedulerService;
 import org.apache.usergrid.management.ApplicationInfo;
 import org.apache.usergrid.management.ManagementService;
 import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.entities.FileImport;
 import org.apache.usergrid.persistence.entities.Import;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.codehaus.jackson.JsonFactory;
@@ -129,6 +127,72 @@ public class ImportServiceImpl implements ImportService {
     }
 
     /**
+     *
+     * @param file  file to be scheduled
+     * @return it returns the UUID of the scheduled job
+     * @throws Exception
+     */
+    @Override
+    public UUID scheduleFile(File file,EntityRef importRef) throws Exception {
+
+        ApplicationInfo defaultImportApp = null;
+
+        EntityManager em = null;
+        Set<String> collections=new HashSet<String>();
+        try {
+            em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+            collections = em.getApplicationCollections();
+            if ( !collections.contains( "importFiles" ) ) {
+                em.createApplicationCollection( "importFiles" );
+            }
+        }
+        catch ( Exception e ) {
+            logger.error( "application doesn't exist within the current context" );
+            return null;
+        }
+
+        RelationManager rm = em.getRelationManager(importRef);
+        FileImport fileImport = new FileImport();
+
+        fileImport.setFileName(file.getName());
+        fileImport.setCompleted(false);
+        fileImport.setLastUpdatedUUID("");
+        fileImport.setErrorMessage("");
+        fileImport.setState(FileImport.State.CREATED);
+
+        fileImport = em.create(fileImport);
+        EntityRef fileRef = em.getRef(fileImport.getUuid());
+
+        //update state
+        try {
+            rm.addToCollection("importFiles",fileRef);
+        }
+        catch ( Exception e ) {
+            logger.error(e.getMessage());
+            return null;
+        }
+        fileImport.setState( FileImport.State.CREATED );
+        em.update( fileImport );
+
+        //set data to be transferred to importInfo
+        JobData jobData = new JobData();
+        jobData.setProperty( "File ", file );
+        jobData.setProperty( "fileImportId", fileImport.getUuid() );
+        jobData.setProperty("importUUID",importRef);
+
+        long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+        //schedule job
+        sch.createJob("fileImportJob", soonestPossible, jobData );
+
+        //update state
+        fileImport.setState( FileImport.State.SCHEDULED );
+        em.update( fileImport );
+
+        return fileImport.getUuid();
+    }
+
+    /**
      * Query Entity Manager for the string state of the Import Entity. This corresponds to the GET /import
      *
      * @return String
@@ -179,7 +243,7 @@ public class ImportServiceImpl implements ImportService {
         return importUG.getErrorMessage().toString();
     }
 
-
+    @Override
     public Import getImportEntity( final JobExecution jobExecution ) throws Exception {
 
         UUID importId = ( UUID ) jobExecution.getJobData().getProperty( IMPORT_ID );
@@ -189,6 +253,15 @@ public class ImportServiceImpl implements ImportService {
     }
 
     @Override
+    public FileImport getFileImportEntity( final JobExecution jobExecution ) throws Exception {
+
+        UUID fileImportId = ( UUID ) jobExecution.getJobData().getProperty("fileImportId");
+        EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+        return em.get( fileImportId, FileImport.class );
+    }
+
+    @Override
     public ArrayList<File> getEphemeralFile() {
         return files;
     }
@@ -248,6 +321,7 @@ public class ImportServiceImpl implements ImportService {
 
         //update the entity state to show that the job has officially started.
         importUG.setState(Import.State.STARTED);
+        importUG.setStarted(System.currentTimeMillis());
         em.update(importUG);
         try {
             if (s3PlaceHolder != null) {
@@ -264,8 +338,10 @@ public class ImportServiceImpl implements ImportService {
         }
 
         try {
+
             if (config.get("organizationId") == null) {
                 logger.error("No organization could be found");
+                importUG.setErrorMessage("No organization could be found");
                 importUG.setState(Import.State.FAILED);
                 em.update(importUG);
                 return;
@@ -279,29 +355,20 @@ public class ImportServiceImpl implements ImportService {
                 //imports a single collection from an app org combo
                 importCollectionFromOrgApp((UUID) config.get("applicationId"), config, jobExecution, s3Import);
             }
-        }
-        catch (OrganizationNotFoundException e) {
-            importUG.setErrorMessage(e.getMessage());
-            importUG.setState(Import.State.FAILED);
-            em.update(importUG);
-            return;
-        }
-        catch (ApplicationNotFoundException e) {
-            importUG.setErrorMessage(e.getMessage());
-            importUG.setState(Import.State.FAILED);
-            em.update(importUG);
+
+            Map<String,Object> FileJobID = new HashMap<String,Object>();
+            for(File eachfile: files) {
+                UUID jobID = scheduleFile(eachfile, em.getRef(importId));
+                FileJobID.put(eachfile.getName(), jobID.toString());
+            }
             return;
         }
         catch (Exception e) {
             // the case where job will be retried i.e. resumed from the failed point
             importUG.setErrorMessage(e.getMessage());
-            importUG.setState(Import.State.FAILED);
             em.update(importUG);
             throw e;
         }
-
-        importUG.setState( Import.State.FINISHED );
-        em.update( importUG );
     }
 
     /**
@@ -324,7 +391,6 @@ public class ImportServiceImpl implements ImportService {
 
         files = fileTransfer( importUG, appFileName, config, s3Import, 0 );
 
-        FileParser(jobExecution);
     }
 
     /**
@@ -346,7 +412,6 @@ public class ImportServiceImpl implements ImportService {
 
         files = fileTransfer( importUG, appFileName, config, s3Import, 1 );
 
-        FileParser(jobExecution);
     }
 
     /**
@@ -367,7 +432,6 @@ public class ImportServiceImpl implements ImportService {
         appFileName = prepareInputFileName( "organization", organizationInfo.getName() , null );
         files = fileTransfer( importUG, appFileName, config, s3Import, 2 );
 
-        FileParser(jobExecution);
     }
 
     /**
@@ -424,91 +488,107 @@ public class ImportServiceImpl implements ImportService {
      * The loops through each temp file and parses it to store the entities from the json back into usergrid
      * @throws Exception
      */
-    private void FileParser(JobExecution jobExecution) throws Exception {
-
-        // add properties to the import entity
-        Import importUG = getImportEntity(jobExecution);
-        EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
-
-
-
-        Map<String,Object> fileMetadata = new HashMap<String, Object>();
-        ArrayList<Map<String,Object>> value = new ArrayList<Map<String, Object>>();
-
-        if (!((Map<String,Object>)importUG.getDynamicProperties()).containsKey("files")) {
-            // create the structure for file metadata and initialize it
-            for (File collectionFile : files) {
-                Map<String, Object> singleFile = new HashMap<String, Object>();
-                singleFile.put("name", collectionFile.getName());
-                singleFile.put("completed", new Boolean(false));
-                singleFile.put("lastUpdatedUUID", new String(""));
-                value.add(singleFile);
-            }
+    @Override
+    public void FileParser(JobExecution jobExecution) throws Exception {
 
-            fileMetadata.put("files", value);
-            importUG.addProperties(fileMetadata);
-            rootEm.update(importUG);
-        }
 
-        ArrayList fileNames = (ArrayList)importUG.getDynamicProperties().get("files");
-        int i=0;
+        // add properties to the import entity
+        FileImport fileImport = getFileImportEntity(jobExecution);
+        File file = (File)jobExecution.getJobData().getProperty("file");
 
-        for(File collectionFile : files) {
+        EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
 
-            Map<String,Object> fileInfo = (Map<String,Object>)fileNames.get(i);
-            boolean completed = ((Boolean)fileInfo.get("completed")).booleanValue();
-            // on resume, completed files will not be traversed again
-            if(!completed) {
+        boolean completed = fileImport.getCompleted();
+        // on resume, completed files will not be traversed again
+        if(!completed) {
 
-                if(!isValidJSON(collectionFile,rootEm,importUG,i)){
-                    i++;
-                    continue;
-                }
+            if (isValidJSON(file, rootEm, fileImport)) {
 
-                String applicationName = collectionFile.getPath().split("\\.")[0];
+                String applicationName = file.getPath().split("\\.")[0];
 
                 ApplicationInfo application = managementService.getApplicationInfo(applicationName);
 
-                JsonParser jp = getJsonParserForFile(collectionFile);
-                String lastUpdatedUUID  = fileInfo.get("lastUpdatedUUID").toString();
+                JsonParser jp = getJsonParserForFile(file);
+                String lastUpdatedUUID = fileImport.getLastUpdatedUUID();
 
                 // this handles partially completed files by updating entities from the point of failure
-                if(!lastUpdatedUUID.equals(""))
-                {
+                if (!lastUpdatedUUID.equals("")) {
                     // go till the last updated entity
-                     while(!jp.getText().equals(lastUpdatedUUID)) {
-                         jp.nextToken();
-                     }
+                    while (!jp.getText().equals(lastUpdatedUUID)) {
+                        jp.nextToken();
+                    }
 
                     // skip the last one and start from teh next one
-                    while(!(jp.getCurrentToken()==JsonToken.END_OBJECT && jp.nextToken() == JsonToken.START_OBJECT)) {
+                    while (!(jp.getCurrentToken() == JsonToken.END_OBJECT && jp.nextToken() == JsonToken.START_OBJECT)) {
                         jp.nextToken();
                     }
                 }
 
                 // get to start of an object i.e next entity.
-                while(jp.getCurrentToken() != JsonToken.START_OBJECT) {
+                while (jp.getCurrentToken() != JsonToken.START_OBJECT) {
                     jp.nextToken();
                 }
 
                 EntityManager em = emf.getEntityManager(application.getId());
+                try {
+                    while (jp.nextToken() != JsonToken.END_ARRAY) {
+                        importEntityStuff(jp, em, rootEm, fileImport);
+                    }
+                    jp.close();
+                }
+                catch (OrganizationNotFoundException e) {
+                    fileImport.setErrorMessage(e.getMessage());
+                    fileImport.setState(FileImport.State.FINISHED);
+                    em.update(fileImport);
+                    return;
+                }
+                catch (ApplicationNotFoundException e) {
+                    fileImport.setErrorMessage(e.getMessage());
+                    fileImport.setState(FileImport.State.FINISHED);
+                    em.update(fileImport);
+                    return;
+                }
+                catch (Exception e) {
+                    // the case where job will be retried i.e. resumed from the failed point
+                    fileImport.setErrorMessage(e.getMessage());
+                    em.update(fileImport);
+                    throw e;
+                }
 
-                //TODO: remove roles and take care of it later when importing applications
-                while (jp.nextToken() != JsonToken.END_ARRAY) {
-                    importEntityStuff(jp, em, rootEm, importUG, i);
+                if(!fileImport.getState().equals("FAILED")) {
+                    // mark file as completed
+                    fileImport.setCompleted(true);
+                    fileImport.setState(FileImport.State.FINISHED);
+                    rootEm.update(fileImport);
+                    //check other files status and mark the status of import Job.
+                    Results filesCollection = rootEm.getCollection(MANAGEMENT_APPLICATION_ID,"importFiles",null, Results.Level.ALL_PROPERTIES);
+                    List<Entity> entities = filesCollection.getEntities();
+                    UUID importId = UUID.fromString(jobExecution.getJobData().getProperty("importUUID").toString());
+                    Import importUG = rootEm.get(importId, Import.class);
+                    int count = 0;
+                    for(Entity eachEntity: entities) {
+
+                        FileImport fi = em.get(eachEntity.getUuid(),FileImport.class);
+                        if(fi.getState().equals("FINISHED")) {
+                            count++;
+                        }
+                        else if(fi.getState().equals("FAILED")) {
+                            importUG.setState(Import.State.FAILED);
+                            rootEm.update(importUG);
+                            break;
+                        }
+                    }
+                    if(count == entities.size()) {
+                       importUG.setState(Import.State.FINISHED);
+                       rootEm.update(importUG);
+                    }
                 }
-                jp.close();
             }
-            // mark file as completed
-            ((Map<String,Object>)fileNames.get(i)).put("completed",true);
-            rootEm.update(importUG);
-            i++;
         }
     }
 
-    private boolean isValidJSON( File collectionFile, EntityManager rootEm, Import importUG, int index) throws Exception  {
+    private boolean isValidJSON( File collectionFile, EntityManager rootEm, FileImport fileImport) throws Exception  {
 
-        ArrayList fileNames = (ArrayList) importUG.getDynamicProperties().get("files");
         boolean valid = false;
         try {
             final JsonParser jp = jsonFactory.createJsonParser(collectionFile);
@@ -517,12 +597,11 @@ public class ImportServiceImpl implements ImportService {
             valid = true;
         } catch (JsonParseException e) {
             e.printStackTrace();
-            ((Map<String, Object>) fileNames.get(index)).put("Error Message", e.getMessage());
-            rootEm.update(importUG);
+            fileImport.setErrorMessage(e.getMessage());
+            rootEm.update(fileImport);
         } catch (IOException e) {
-            e.printStackTrace();
-            ((Map<String, Object>) fileNames.get(index)).put("Miscellaneous error", e.getMessage());
-            rootEm.update(importUG);
+            fileImport.setErrorMessage(e.getMessage());
+            rootEm.update(fileImport);
         }
         return valid;
     }
@@ -538,9 +617,8 @@ public class ImportServiceImpl implements ImportService {
      *
      * @param jp JsonPrser pointing to the beginning of the object.
      */
-    private void importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, Import importUG, int index) throws Exception {
+    private void importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, FileImport fileImport) throws Exception {
 
-        ArrayList fileNames = (ArrayList) importUG.getDynamicProperties().get("files");
         Entity entity = null;
         EntityRef ownerEntityRef=null;
         String entityUuid="";
@@ -613,21 +691,22 @@ public class ImportServiceImpl implements ImportService {
             }
             catch (IllegalArgumentException e) {
                 // skip illegal entity UUID and go to next one
-                ((Map<String, Object>) fileNames.get(index)).put("Entity Creation Error", e.getMessage());
-                rootEm.update(importUG);
+                fileImport.setErrorMessage(e.getMessage());
+                rootEm.update(fileImport);
             }
             catch (Exception e) {
                 // skip illegal entity UUID and go to next one
-                ((Map<String, Object>) fileNames.get(index)).put("Miscellaneous Error", e.getMessage());
-                rootEm.update(importUG);
+                fileImport.setErrorMessage(e.getMessage());
+                rootEm.update(fileImport);
             }
         }
+
         // update the last updated entity
         if(entity != null) {
             entityCount++;
-            if(entityCount == 50) {
-                ((Map<String, Object>) fileNames.get(index)).put("lastUpdatedUUID", entityUuid);
-                rootEm.update(importUG);
+            if(entityCount == 1000) {
+                fileImport.setLastUpdatedUUID(entityUuid);
+                rootEm.update(fileImport);
                 entityCount = 0;
             }
         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index fc049fc..dc06dea 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -96,5 +96,6 @@
   </bean>
 
   <bean id="importJob" class="org.apache.usergrid.management.importUG.ImportJob" />
+  <bean id="fileImportJob" class="org.apache.usergrid.management.importUG.FileImportJob" />
 
 </beans>