You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/22 16:16:28 UTC
[92/95] [abbrv] git commit: added logic for updating lastupdatedUUID
based on event logic
added logic for updating lastupdatedUUID based on event logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3b565609
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3b565609
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3b565609
Branch: refs/heads/import-feature
Commit: 3b56560953097409f83e171e182683d01b04da25
Parents: 6f80f6a
Author: Pooja Jain <pj...@apigee.com>
Authored: Thu Aug 7 16:45:25 2014 -0700
Committer: Pooja Jain <pj...@apigee.com>
Committed: Thu Aug 7 16:45:25 2014 -0700
----------------------------------------------------------------------
.../management/importUG/ImportServiceImpl.java | 53 +++++++++++++-------
1 file changed, 35 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3b565609/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
index 0707fef..e650798 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importUG/ImportServiceImpl.java
@@ -42,6 +42,7 @@ import rx.schedulers.Schedulers;
import java.io.File;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
@@ -691,7 +692,7 @@ public class ImportServiceImpl implements ImportService {
* @param jobExecution execution details for the import jbo
* @throws Exception
*/
- private void importEntityStuff(final JsonParser jp, final EntityManager em, EntityManager rootEm, final FileImport fileImport, final JobExecution jobExecution) throws Exception {
+ private void importEntityStuff(final JsonParser jp, final EntityManager em, final EntityManager rootEm, final FileImport fileImport, final JobExecution jobExecution) throws Exception {
final JsonParserObservable subscribe = new JsonParserObservable(jp, em, rootEm, fileImport);
@@ -709,6 +710,8 @@ public class ImportServiceImpl implements ImportService {
};
+ final AtomicLong entityCounter = new AtomicLong();
+ final AtomicLong eventCounter = new AtomicLong();
/**
* This is boilerplate glue code. We have to follow this for the parallel operation. In the "call"
* method we want to simply return the input observable + the chain of operations we want to invoke
@@ -716,11 +719,34 @@ public class ImportServiceImpl implements ImportService {
observable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@Override
public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
- return entityWrapperObservable.doOnNext(doWork);
+ return entityWrapperObservable.doOnNext(doWork).doOnNext(new Action1<WriteEvent>() {
+
+ @Override
+ public void call(WriteEvent writeEvent) {
+ if (!(writeEvent instanceof EntityEvent)) {
+ final long val = eventCounter.incrementAndGet();
+ if(val % 50 == 0) {
+ jobExecution.heartbeat();
+ }
+ return;
+ }
+
+ final long value = entityCounter.incrementAndGet();
+ if (value % 2000 == 0) {
+ try {
+ fileImport.setLastUpdatedUUID(((EntityEvent) writeEvent).getEntityUuid().toString());
+ //checkpoint the UUID here.
+ rootEm.update(fileImport);
+ } catch(Exception ex) {}
+ }
+ if(value % 100 == 0) {
+ jobExecution.heartbeat();
+ }
+ }
+ }
+ );
}
}, Schedulers.io()).toBlocking().last();
-
-
}
private interface WriteEvent {
@@ -738,25 +764,17 @@ public class ImportServiceImpl implements ImportService {
this.properties = properties;
}
+ public UUID getEntityUuid() {
+ return entityUuid;
+ }
+
// Creates entities
@Override
public void doWrite(EntityManager em, JobExecution jobExecution, FileImport fileImport) {
EntityManager rootEm = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
try {
- Entity entity = em.create(entityUuid, entityType, properties);
- // update the last updated entity
- if (entity != null) {
- entityCount++;
- if ((entityCount % 10) == 0) {
- jobExecution.heartbeat();
- }
- if (entityCount == 2000) {
- fileImport.setLastUpdatedUUID(entity.getUuid().toString());
- rootEm.update(fileImport);
- entityCount = 0;
- }
- }
+ em.create(entityUuid, entityType, properties);
} catch (Exception e) {
fileImport.setErrorMessage(e.getMessage());
try {
@@ -786,7 +804,6 @@ public class ImportServiceImpl implements ImportService {
try {
em.createConnection(ownerEntityRef, connectionType, entryRef);
-
} catch (Exception e) {
fileImport.setErrorMessage(e.getMessage());
try {