You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/22 16:16:15 UTC
[79/95] [abbrv] git commit: added heartbeat and logging information
added heartbeat and logging information
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6b00be7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6b00be7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6b00be7a
Branch: refs/heads/import-feature
Commit: 6b00be7abd5dc95152bee5be87cf226e4639084f
Parents: 9a00e58
Author: Pooja Jain <pj...@apigee.com>
Authored: Mon Aug 4 20:46:08 2014 -0700
Committer: Pooja Jain <pj...@apigee.com>
Committed: Mon Aug 4 20:46:08 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/batch/Job.java | 3 ++-
.../usergrid/management/importUG/FileImportJob.java | 1 +
.../usergrid/management/importUG/ImportJob.java | 1 +
.../management/importUG/ImportServiceImpl.java | 16 +++++++++++-----
4 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b00be7a/stack/core/src/main/java/org/apache/usergrid/batch/Job.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/Job.java b/stack/core/src/main/java/org/apache/usergrid/batch/Job.java
index 3a35f49..dbd034f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/Job.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/Job.java
@@ -37,7 +37,8 @@ public interface Job {
/**
* Invoked when a job is marked as dead by the scheduler. In some instances, jobs need to know
- * this information to handle this case appropriately
+ * this information to handle this case appropriately. Dead is defined as the retry count has been
+ * exceeded. I.E 10 failures allowed, this is the 11th attempt to start.
*
* @param execution
* @throws Exception
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b00be7a/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
index b03b0ae..f4d58ff 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/FileImportJob.java
@@ -74,6 +74,7 @@ public class FileImportJob extends OnlyOnceJob {
// logger.error(e.getMessage());
// throw e;
// }
+ logger.error("File Import Service completed job");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b00be7a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
index 6a9a999..3f02ebd 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportJob.java
@@ -68,6 +68,7 @@ public class ImportJob extends OnlyOnceJob {
// logger.error(e.getMessage());
// throw e;
// }
+ logger.error("Import Service completed job");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b00be7a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
index 5881528..c35f843 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
@@ -509,6 +509,7 @@ public class ImportServiceImpl implements ImportService {
public void FileParser(JobExecution jobExecution) throws Exception {
+
// add properties to the import entity
FileImport fileImport = getFileImportEntity(jobExecution);
@@ -519,6 +520,8 @@ public class ImportServiceImpl implements ImportService {
boolean completed = fileImport.getCompleted();
+ logger.error("File parser called for " + file.getName());
+
// on resume, completed files will not be traversed again
if(!completed) {
@@ -528,14 +531,11 @@ public class ImportServiceImpl implements ImportService {
String applicationName = file.getPath().split("\\.")[0];
- logger.error(applicationName);
-
ApplicationInfo application = managementService.getApplicationInfo(applicationName);
JsonParser jp = getJsonParserForFile(file);
String lastUpdatedUUID = fileImport.getLastUpdatedUUID();
- logger.error(lastUpdatedUUID);
// this handles partially completed files by updating entities from the point of failure
if (!lastUpdatedUUID.equals(" ")) {
// go till the last updated entity
@@ -559,7 +559,7 @@ public class ImportServiceImpl implements ImportService {
String uuid=" ";
while (jp.nextToken() != JsonToken.END_ARRAY) {
- uuid = importEntityStuff(jp, em, rootEm, fileImport);
+ uuid = importEntityStuff(jp, em, rootEm, fileImport, jobExecution);
}
//one file completed
@@ -591,6 +591,9 @@ public class ImportServiceImpl implements ImportService {
// }
if(!fileImport.getState().equals("FAILED")) {
+
+ logger.error("File parsing completed called for " + file.getName());
+
// mark file as completed
fileImport.setCompleted(true);
fileImport.setState(FileImport.State.FINISHED);
@@ -660,7 +663,7 @@ public class ImportServiceImpl implements ImportService {
*
* @param jp JsonPrser pointing to the beginning of the object.
*/
- private String importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, FileImport fileImport) throws Exception {
+ private String importEntityStuff( JsonParser jp, EntityManager em, EntityManager rootEm, FileImport fileImport, JobExecution jobExecution) throws Exception {
Entity entity = null;
EntityRef ownerEntityRef = null;
@@ -745,6 +748,9 @@ public class ImportServiceImpl implements ImportService {
// update the last updated entity
if (entity != null) {
entityCount++;
+ if( (entityCount%10) == 0) {
+ jobExecution.heartbeat();
+ }
if (entityCount == 1000) {
fileImport.setLastUpdatedUUID(entityUuid);
rootEm.update(fileImport);