You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/22 16:16:02 UTC
[66/95] [abbrv] git commit: - Added implementation for distributing
jobs file wise
- Added implementation for distributing jobs file wise
TO DO:
- test the above
- Add endpoint to get file Job information as now the current will only give overall job status
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a26716d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a26716d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a26716d2
Branch: refs/heads/import-feature
Commit: a26716d23b9f7cb281b752c28ecba0c605df6c99
Parents: cc0ad6e
Author: Pooja Jain <pj...@apigee.com>
Authored: Tue Jul 29 18:59:51 2014 -0700
Committer: Pooja Jain <pj...@apigee.com>
Committed: Tue Jul 29 18:59:51 2014 -0700
----------------------------------------------------------------------
.../persistence/entities/FileImport.java | 120 +++++++++
.../management/importUG/FileImportJob.java | 96 +++++++
.../usergrid/management/importUG/ImportJob.java | 19 ++
.../management/importUG/ImportService.java | 31 +++
.../management/importUG/ImportServiceImpl.java | 263 ++++++++++++-------
.../resources/usergrid-services-context.xml | 1 +
6 files changed, 438 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
new file mode 100644
index 0000000..8688bec
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.entities;
+
+
+import org.apache.usergrid.persistence.TypedEntity;
+import org.apache.usergrid.persistence.annotations.EntityProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+
+/**
+ * Contains state information for an Entity Job
+ */
+@XmlRootElement
+public class FileImport extends TypedEntity {
+ //canceled , and expired states aren't used in current iteration.
+ public static enum State {
+ CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
+ }
+
+ @EntityProperty
+ protected State curState;
+
+ /**
+ * Error message
+ */
+ @EntityProperty
+ protected String errorMessage;
+
+
+ /**
+ * file name
+ */
+ @EntityProperty
+ protected String fileName;
+
+ /**
+ * File completion Status
+ */
+ @EntityProperty
+ protected Boolean completed;
+
+ /**
+ * LastUpdatedUUID
+ */
+ @EntityProperty
+ protected String lastUpdatedUUID;
+
+ public FileImport() {
+ }
+
+
+ public String getLastUpdatedUUID() {
+ return lastUpdatedUUID;
+ }
+
+
+ public void setLastUpdatedUUID( final String lastUpdatedUUID ) {
+ this.lastUpdatedUUID = lastUpdatedUUID;
+ }
+
+ public Boolean getCompleted() {
+ return completed;
+ }
+
+
+ public void setCompleted( final Boolean completed ) {
+ this.completed = completed;
+ }
+
+ //state should moved to a derived state, but it is not there yet.
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public void setState( State setter ) {
+ curState = setter;
+ }
+
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public State getState() { return curState; }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+
+ public void setErrorMessage( final String errorMessage ) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+
+ public void setFileName( final String fileName ) {
+ this.fileName = fileName;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
new file mode 100644
index 0000000..1e14116
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.management.importUG;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.FileImport;
+import org.apache.usergrid.persistence.entities.JobData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+
+/**
+ * Created by ApigeeCorporation on 7/8/14.
+ */
+@Component("fileImportJob")
+public class FileImportJob extends OnlyOnceJob {
+
+ public static final String FILE_IMPORT_ID = "fileImportId";
+ private static final Logger logger = LoggerFactory.getLogger(FileImportJob.class);
+
+ @Autowired
+ ImportService importService;
+
+ //injected the Entity Manager Factory
+ protected EntityManagerFactory emf;
+
+ public FileImportJob() {
+ logger.info( "FileImportJob created " + this );
+ }
+
+ @Override
+ protected void doJob(JobExecution jobExecution) throws Exception {
+ logger.info( "execute FileImportJob {}", jobExecution );
+
+ JobData jobData = jobExecution.getJobData();
+ if ( jobData == null ) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+
+ jobExecution.heartbeat();
+ try {
+ importService.FileParser( jobExecution );
+ }
+ catch ( Exception e ) {
+ logger.error( " Import Service failed to complete file import job" );
+ logger.error(e.getMessage());
+ throw e;
+ }
+
+ logger.info( "executed FileImportJob process completed" );
+ }
+
+ @Override
+ protected long getDelay(JobExecution execution) throws Exception {
+ return 100;
+ }
+
+ @Autowired
+ public void setImportService( final ImportService importService ) {
+ this.importService = importService;
+ }
+
+ @Override
+ public void dead( final JobExecution execution ) throws Exception {
+
+ EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+ FileImport fileImport = importService.getFileImportEntity(execution);
+ fileImport.setErrorMessage("The Job has been tried maximum times but still failed");
+ fileImport.setState(FileImport.State.FAILED);
+ rootEm.update(fileImport);
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
index db0c4c8..3b1a0f2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
@@ -19,12 +19,17 @@ package org.apache.usergrid.management.importUG;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.entities.JobData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+
/**
* Created by ApigeeCorporation on 7/8/14.
*/
@@ -37,6 +42,9 @@ public class ImportJob extends OnlyOnceJob {
@Autowired
ImportService importService;
+ //injected the Entity Manager Factory
+ protected EntityManagerFactory emf;
+
public ImportJob() {
logger.info( "ImportJob created " + this );
}
@@ -73,4 +81,15 @@ public class ImportJob extends OnlyOnceJob {
public void setImportService( final ImportService importService ) {
this.importService = importService;
}
+
+ @Override
+ public void dead( final JobExecution execution ) throws Exception {
+
+ EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+ Import importUG = importService.getImportEntity(execution);
+ importUG.setErrorMessage("The Job has been tried maximum times but still failed");
+ importUG.setState(Import.State.FAILED);
+ rootEm.update(importUG);
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
index 6866036..d0be747 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportService.java
@@ -18,6 +18,9 @@
package org.apache.usergrid.management.importUG;
import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.entities.FileImport;
+import org.apache.usergrid.persistence.entities.Import;
import java.io.File;
import java.util.ArrayList;
@@ -35,11 +38,23 @@ public interface ImportService {
UUID schedule(Map<String, Object> json) throws Exception;
/**
+ * Schedules the import to execute
+ */
+ UUID scheduleFile(File fileName,EntityRef importRef) throws Exception;
+
+ /**
* Perform the import from the external resource
*/
void doImport(JobExecution jobExecution) throws Exception;
/**
+ * Parses the input file and creates entities
+ * @param jobExecution
+ * @throws Exception
+ */
+ void FileParser(JobExecution jobExecution) throws Exception;
+
+ /**
* Returns the current state of the service.
*/
String getState(UUID state) throws Exception;
@@ -51,4 +66,20 @@ public interface ImportService {
*/
ArrayList<File> getEphemeralFile();
+ /**
+ *
+ * @param jobExecution
+ * @return FileImportEntity
+ * @throws Exception
+ */
+ FileImport getFileImportEntity( final JobExecution jobExecution ) throws Exception;
+ /**
+ *
+ * @param jobExecution
+ * @return getImportEntity
+ * @throws Exception
+ */
+ Import getImportEntity( final JobExecution jobExecution ) throws Exception;
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
index 59281b4..340b058 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
@@ -22,10 +22,8 @@ import org.apache.usergrid.batch.service.SchedulerService;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.entities.FileImport;
import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.entities.JobData;
import org.codehaus.jackson.JsonFactory;
@@ -129,6 +127,72 @@ public class ImportServiceImpl implements ImportService {
}
/**
+ *
+ * @param file file to be scheduled
+ * @return it returns the UUID of the scheduled job
+ * @throws Exception
+ */
+ @Override
+ public UUID scheduleFile(File file,EntityRef importRef) throws Exception {
+
+ ApplicationInfo defaultImportApp = null;
+
+ EntityManager em = null;
+ Set<String> collections=new HashSet<String>();
+ try {
+ em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ collections = em.getApplicationCollections();
+ if ( !collections.contains( "importFiles" ) ) {
+ em.createApplicationCollection( "importFiles" );
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "application doesn't exist within the current context" );
+ return null;
+ }
+
+ RelationManager rm = em.getRelationManager(importRef);
+ FileImport fileImport = new FileImport();
+
+ fileImport.setFileName(file.getName());
+ fileImport.setCompleted(false);
+ fileImport.setLastUpdatedUUID("");
+ fileImport.setErrorMessage("");
+ fileImport.setState(FileImport.State.CREATED);
+
+ fileImport = em.create(fileImport);
+ EntityRef fileRef = em.getRef(fileImport.getUuid());
+
+ //update state
+ try {
+ rm.addToCollection("importFiles",fileRef);
+ }
+ catch ( Exception e ) {
+ logger.error(e.getMessage());
+ return null;
+ }
+ fileImport.setState( FileImport.State.CREATED );
+ em.update( fileImport );
+
+ //set data to be transferred to importInfo
+ JobData jobData = new JobData();
+ jobData.setProperty( "File ", file );
+ jobData.setProperty( "fileImportId", fileImport.getUuid() );
+ jobData.setProperty("importUUID",importRef);
+
+ long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+ //schedule job
+ sch.createJob("fileImportJob", soonestPossible, jobData );
+
+ //update state
+ fileImport.setState( FileImport.State.SCHEDULED );
+ em.update( fileImport );
+
+ return fileImport.getUuid();
+ }
+
+ /**
* Query Entity Manager for the string state of the Import Entity. This corresponds to the GET /import
*
* @return String
@@ -179,7 +243,7 @@ public class ImportServiceImpl implements ImportService {
return importUG.getErrorMessage().toString();
}
-
+ @Override
public Import getImportEntity( final JobExecution jobExecution ) throws Exception {
UUID importId = ( UUID ) jobExecution.getJobData().getProperty( IMPORT_ID );
@@ -189,6 +253,15 @@ public class ImportServiceImpl implements ImportService {
}
@Override
+ public FileImport getFileImportEntity( final JobExecution jobExecution ) throws Exception {
+
+ UUID fileImportId = ( UUID ) jobExecution.getJobData().getProperty("fileImportId");
+ EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+ return em.get( fileImportId, FileImport.class );
+ }
+
+ @Override
public ArrayList<File> getEphemeralFile() {
return files;
}
@@ -248,6 +321,7 @@ public class ImportServiceImpl implements ImportService {
//update the entity state to show that the job has officially started.
importUG.setState(Import.State.STARTED);
+ importUG.setStarted(System.currentTimeMillis());
em.update(importUG);
try {
if (s3PlaceHolder != null) {
@@ -264,8 +338,10 @@ public class ImportServiceImpl implements ImportService {
}
try {
+
if (config.get("organizationId") == null) {
logger.error("No organization could be found");
+ importUG.setErrorMessage("No organization could be found");
importUG.setState(Import.State.FAILED);
em.update(importUG);
return;
@@ -279,29 +355,20 @@ public class ImportServiceImpl implements ImportService {
//imports a single collection from an app org combo
importCollectionFromOrgApp((UUID) config.get("applicationId"), config, jobExecution, s3Import);
}
- }
- catch (OrganizationNotFoundException e) {
- importUG.setErrorMessage(e.getMessage());
- importUG.setState(Import.State.FAILED);
- em.update(importUG);
- return;
- }
- catch (ApplicationNotFoundException e) {
- importUG.setErrorMessage(e.getMessage());
- importUG.setState(Import.State.FAILED);
- em.update(importUG);
+
+ Map<String,Object> FileJobID = new HashMap<String,Object>();
+ for(File eachfile: files) {
+ UUID jobID = scheduleFile(eachfile, em.getRef(importId));
+ FileJobID.put(eachfile.getName(), jobID.toString());
+ }
return;
}
catch (Exception e) {
// the case where job will be retried i.e. resumed from the failed point
importUG.setErrorMessage(e.getMessage());
- importUG.setState(Import.State.FAILED);
em.update(importUG);
throw e;
}
-
- importUG.setState( Import.State.FINISHED );
- em.update( importUG );
}
/**
@@ -324,7 +391,6 @@ public class ImportServiceImpl implements ImportService {
files = fileTransfer( importUG, appFileName, config, s3Import, 0 );
- FileParser(jobExecution);
}
/**
@@ -346,7 +412,6 @@ public class ImportServiceImpl implements ImportService {
files = fileTransfer( importUG, appFileName, config, s3Import, 1 );
- FileParser(jobExecution);
}
/**
@@ -367,7 +432,6 @@ public class ImportServiceImpl implements ImportService {
appFileName = prepareInputFileName( "organization", organizationInfo.getName() , null );
files = fileTransfer( importUG, appFileName, config, s3Import, 2 );
- FileParser(jobExecution);
}
/**
@@ -424,91 +488,107 @@ public class ImportServiceImpl implements ImportService {
* The loops through each temp file and parses it to store the entities from the json back into usergrid
* @throws Exception
*/
- private void FileParser(JobExecution jobExecution) throws Exception {
-
- // add properties to the import entity
- Import importUG = getImportEntity(jobExecution);
- EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
-
-
-
- Map<String,Object> fileMetadata = new HashMap<String, Object>();
- ArrayList<Map<String,Object>> value = new ArrayList<Map<String, Object>>();
-
- if (!((Map<String,Object>)importUG.getDynamicProperties()).containsKey("files")) {
- // create the structure for file metadata and initialize it
- for (File collectionFile : files) {
- Map<String, Object> singleFile = new HashMap<String, Object>();
- singleFile.put("name", collectionFile.getName());
- singleFile.put("completed", new Boolean(false));
- singleFile.put("lastUpdatedUUID", new String(""));
- value.add(singleFile);
- }
+ @Override
+ public void FileParser(JobExecution jobExecution) throws Exception {
- fileMetadata.put("files", value);
- importUG.addProperties(fileMetadata);
- rootEm.update(importUG);
- }
- ArrayList fileNames = (ArrayList)importUG.getDynamicProperties().get("files");
- int i=0;
+ // add properties to the import entity
+ FileImport fileImport = getFileImportEntity(jobExecution);
+ File file = (File)jobExecution.getJobData().getProperty("file");
- for(File collectionFile : files) {
+ EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
- Map<String,Object> fileInfo = (Map<String,Object>)fileNames.get(i);
- boolean completed = ((Boolean)fileInfo.get("completed")).booleanValue();
- // on resume, completed files will not be traversed again
- if(!completed) {
+ boolean completed = fileImport.getCompleted();
+ // on resume, completed files will not be traversed again
+ if(!completed) {
- if(!isValidJSON(collectionFile,rootEm,importUG,i)){
- i++;
- continue;
- }
+ if (isValidJSON(file, rootEm, fileImport)) {
- String applicationName = collectionFile.getPath().split("\\.")[0];
+ String applicationName = file.getPath().split("\\.")[0];
ApplicationInfo application = managementService.getApplicationInfo(applicationName);
- JsonParser jp = getJsonParserForFile(collectionFile);
- String lastUpdatedUUID = fileInfo.get("lastUpdatedUUID").toString();
+ JsonParser jp = getJsonParserForFile(file);
+ String lastUpdatedUUID = fileImport.getLastUpdatedUUID();
// this handles partially completed files by updating entities from the point of failure
- if(!lastUpdatedUUID.equals(""))
- {
+ if (!lastUpdatedUUID.equals("")) {
// go till the last updated entity
- while(!jp.getText().equals(lastUpdatedUUID)) {
- jp.nextToken();
- }
+ while (!jp.getText().equals(lastUpdatedUUID)) {
+ jp.nextToken();
+ }
// skip the last one and start from teh next one
- while(!(jp.getCurrentToken()==JsonToken.END_OBJECT && jp.nextToken() == JsonToken.START_OBJECT)) {
+ while (!(jp.getCurrentToken() == JsonToken.END_OBJECT && jp.nextToken() == JsonToken.START_OBJECT)) {
jp.nextToken();
}
}
// get to start of an object i.e next entity.
- while(jp.getCurrentToken() != JsonToken.START_OBJECT) {
+ while (jp.getCurrentToken() != JsonToken.START_OBJECT) {
jp.nextToken();
}
EntityManager em = emf.getEntityManager(application.getId());
+ try {
+ while (jp.nextToken() != JsonToken.END_ARRAY) {
+ importEntityStuff(jp, em, rootEm, fileImport);
+ }
+ jp.close();
+ }
+ catch (OrganizationNotFoundException e) {
+ fileImport.setErrorMessage(e.getMessage());
+ fileImport.setState(FileImport.State.FINISHED);
+ em.update(fileImport);
+ return;
+ }
+ catch (ApplicationNotFoundException e) {
+ fileImport.setErrorMessage(e.getMessage());
+ fileImport.setState(FileImport.State.FINISHED);
+ em.update(fileImport);
+ return;
+ }
+ catch (Exception e) {
+ // the case where job will be retried i.e. resumed from the failed point
+ fileImport.setErrorMessage(e.getMessage());
+ em.update(fileImport);
+ throw e;
+ }
- //TODO: remove roles and take care of it later when importing applications
- while (jp.nextToken() != JsonToken.END_ARRAY) {
- importEntityStuff(jp, em, rootEm, importUG, i);
+ if(!fileImport.getState().equals("FAILED")) {
+ // mark file as completed
+ fileImport.setCompleted(true);
+ fileImport.setState(FileImport.State.FINISHED);
+ rootEm.update(fileImport);
+ //check other files status and mark the status of import Job.
+ Results filesCollection = rootEm.getCollection(MANAGEMENT_APPLICATION_ID,"importFiles",null, Results.Level.ALL_PROPERTIES);
+ List<Entity> entities = filesCollection.getEntities();
+ UUID importId = UUID.fromString(jobExecution.getJobData().getProperty("importUUID").toString());
+ Import importUG = rootEm.get(importId, Import.class);
+ int count = 0;
+ for(Entity eachEntity: entities) {
+
+ FileImport fi = em.get(eachEntity.getUuid(),FileImport.class);
+ if(fi.getState().equals("FINISHED")) {
+ count++;
+ }
+ else if(fi.getState().equals("FAILED")) {
+ importUG.setState(Import.State.FAILED);
+ rootEm.update(importUG);
+ break;
+ }
+ }
+ if(count == entities.size()) {
+ importUG.setState(Import.State.FINISHED);
+ rootEm.update(importUG);
+ }
}
- jp.close();
}
- // mark file as completed
- ((Map<String,Object>)fileNames.get(i)).put("completed",true);
- rootEm.update(importUG);
- i++;
}
}
- private boolean isValidJSON( File collectionFile, EntityManager rootEm, Import importUG, int index) throws Exception {
+ private boolean isValidJSON( File collectionFile, EntityManager rootEm, FileImport fileImport) throws Exception {
- ArrayList fileNames = (ArrayList) importUG.getDynamicProperties().get("files");
boolean valid = false;
try {
final JsonParser jp = jsonFactory.createJsonParser(collectionFile);
@@ -517,12 +597,11 @@ public class ImportServiceImpl implements ImportService {
valid = true;
} catch (JsonParseException e) {
e.printStackTrace();
- ((Map<String, Object>) fileNames.get(index)).put("Error Message", e.getMessage());
- rootEm.update(importUG);
+ fileImport.setErrorMessage(e.getMessage());
+ rootEm.update(fileImport);
} catch (IOException e) {
- e.printStackTrace();
- ((Map<String, Object>) fileNames.get(index)).put("Miscellaneous error", e.getMessage());
- rootEm.update(importUG);
+ fileImport.setErrorMessage(e.getMessage());
+ rootEm.update(fileImport);
}
return valid;
}
@@ -538,9 +617,8 @@ public class ImportServiceImpl implements ImportService {
*
* @param jp JsonPrser pointing to the beginning of the object.
*/
- private void importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, Import importUG, int index) throws Exception {
+ private void importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, FileImport fileImport) throws Exception {
- ArrayList fileNames = (ArrayList) importUG.getDynamicProperties().get("files");
Entity entity = null;
EntityRef ownerEntityRef=null;
String entityUuid="";
@@ -613,21 +691,22 @@ public class ImportServiceImpl implements ImportService {
}
catch (IllegalArgumentException e) {
// skip illegal entity UUID and go to next one
- ((Map<String, Object>) fileNames.get(index)).put("Entity Creation Error", e.getMessage());
- rootEm.update(importUG);
+ fileImport.setErrorMessage(e.getMessage());
+ rootEm.update(fileImport);
}
catch (Exception e) {
// skip illegal entity UUID and go to next one
- ((Map<String, Object>) fileNames.get(index)).put("Miscellaneous Error", e.getMessage());
- rootEm.update(importUG);
+ fileImport.setErrorMessage(e.getMessage());
+ rootEm.update(fileImport);
}
}
+
// update the last updated entity
if(entity != null) {
entityCount++;
- if(entityCount == 50) {
- ((Map<String, Object>) fileNames.get(index)).put("lastUpdatedUUID", entityUuid);
- rootEm.update(importUG);
+ if(entityCount == 1000) {
+ fileImport.setLastUpdatedUUID(entityUuid);
+ rootEm.update(fileImport);
entityCount = 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a26716d2/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index fc049fc..dc06dea 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -96,5 +96,6 @@
</bean>
<bean id="importJob" class="org.apache.usergrid.management.importUG.ImportJob" />
+ <bean id="fileImportJob" class="org.apache.usergrid.management.importUG.FileImportJob" />
</beans>