You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/11 00:41:05 UTC
[1/2] git commit: Use EntityManager instead of
EntityCollectionManager to repersist.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 884b0300d -> 56d703898
Use EntityManager instead of EntityCollectionManager to repersist.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a3ece54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a3ece54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a3ece54
Branch: refs/heads/two-dot-o
Commit: 5a3ece54eb81859d9ce40f98416e7fd8239fc6ef
Parents: 693cb2d
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 10 18:14:54 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 10 18:14:54 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 87 ++-----
.../corepersistence/CpEntityManagerFactory.java | 22 --
.../usergrid/corepersistence/CpVisitor.java | 8 +-
.../usergrid/corepersistence/CpWalker.java | 144 +++--------
.../HybridEntityManagerFactory.java | 10 -
.../corepersistence/results/EntityVerifier.java | 3 +-
.../results/FilteringLoader.java | 7 +-
.../usergrid/persistence/EntityManager.java | 3 -
.../persistence/EntityManagerFactory.java | 4 -
.../cassandra/EntityManagerFactoryImpl.java | 10 -
.../cassandra/EntityManagerImpl.java | 7 +-
.../PerformanceEntityRepersistTest.java | 236 -------------------
.../impl/EntityCollectionManagerImpl.java | 36 +--
stack/pom.xml | 4 +-
.../apache/usergrid/rest/SystemResource.java | 30 ++-
.../org/apache/usergrid/tools/RepersistAll.java | 98 --------
16 files changed, 98 insertions(+), 611 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 548afda..88cee9c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2862,85 +2862,30 @@ public class CpEntityManager implements EntityManager {
@Override
public void visitCollectionEntry(
- EntityCollectionManager ecm,
- String collectionName,
- org.apache.usergrid.persistence.model.entity.Entity collectionEntity,
- org.apache.usergrid.persistence.model.entity.Entity memberEntity) {
-
- EntityRef source = new SimpleEntityRef(
- collectionEntity.getId().getType(), collectionEntity.getId().getUuid() );
- EntityRef target = new SimpleEntityRef(
- memberEntity.getId().getType(), memberEntity.getId().getUuid() );
- po.onProgress(source, target, "dummy");
-
- indexEntityIntoCollection(
- collectionEntity, memberEntity, collectionName );
- }
-
- @Override
- public void visitConnectionEntry(
- EntityCollectionManager ecm,
- String connectionType,
- org.apache.usergrid.persistence.model.entity.Entity sourceEntity,
- org.apache.usergrid.persistence.model.entity.Entity targetEntity) {
-
- EntityRef source = new SimpleEntityRef(
- sourceEntity.getId().getType(), sourceEntity.getId().getUuid() );
- EntityRef target = new SimpleEntityRef(
- targetEntity.getId().getType(), targetEntity.getId().getUuid() );
- po.onProgress(source, target, "dummy");
-
- indexEntityIntoConnection(
- sourceEntity, targetEntity, connectionType );
- }
- });
- }
-
-
- /**
- * Completely repersist the application associated with this EntityManager.
- */
- public void repersistApplication(
- final UUID appId, final EntityManagerFactory.ProgressObserver po ) throws Exception {
+ EntityManager em, String collName, Entity source, Entity target) {
- CpWalker walker = new CpWalker();
+ try {
+ em.update( target);
+ po.onProgress(source, target, collName);
- walker.walkCollections( this, application, new CpVisitor() {
+ } catch (Exception ex) {
+ logger.error("Error repersisting entity", ex);
+ }
+ }
@Override
- public void visitCollectionEntry(
- EntityCollectionManager ecm,
- String collectionName,
- org.apache.usergrid.persistence.model.entity.Entity collectionEntity,
- org.apache.usergrid.persistence.model.entity.Entity memberEntity) {
-
- EntityRef source = new SimpleEntityRef(
- collectionEntity.getId().getType(), collectionEntity.getId().getUuid() );
- EntityRef target = new SimpleEntityRef(
- memberEntity.getId().getType(), memberEntity.getId().getUuid() );
- po.onProgress(source, target, "dummy");
+ public void visitConnectionEntry(
+ EntityManager em, String connType, Entity source, Entity target) {
- ecm.write( memberEntity).toBlocking().last();
+ try {
+ em.update( target);
+ po.onProgress(source, target, connType);
- indexEntityIntoCollection(
- collectionEntity, memberEntity, collectionName );
+ } catch (Exception ex) {
+ logger.error("Error repersisting entity", ex);
+ }
}
- @Override
- public void visitConnectionEntry(
- EntityCollectionManager ecm,
- String connectionType,
- org.apache.usergrid.persistence.model.entity.Entity sourceEntity,
- org.apache.usergrid.persistence.model.entity.Entity targetEntity) {
-
- EntityRef source = new SimpleEntityRef(
- sourceEntity.getId().getType(), sourceEntity.getId().getUuid() );
- EntityRef target = new SimpleEntityRef(
- targetEntity.getId().getType(), targetEntity.getId().getUuid() );
- po.onProgress(source, target, "dummy");
-
- ecm.write( targetEntity).toBlocking().last();
- }
});
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index cf19050..1048505 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -699,26 +699,4 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
throw new UnsupportedOperationException( "Not supported yet." );
}
-
- @Override
- public void repersistApplication( UUID appId, ProgressObserver po ) throws Exception {
- EntityManager em = getEntityManager( appId );
- em.repersistApplication( appId, po );
- }
-
- @Override
- public void repersistAll( ProgressObserver po ) throws Exception {
-
- logger.info("\n\nRepersisting all Entities\n");
-
- Map<String, UUID> appMap = getApplications();
-
- logger.info("About to repersist entities for {} applications", appMap.keySet().size());
-
- for ( UUID appUuid : appMap.values() ) {
- repersistApplication( appUuid, po );
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
index 02d13c1..5d32235 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
@@ -15,8 +15,8 @@
*/
package org.apache.usergrid.corepersistence;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
/**
@@ -25,8 +25,8 @@ import org.apache.usergrid.persistence.model.entity.Entity;
public interface CpVisitor {
public void visitCollectionEntry(
- EntityCollectionManager ecm, String collName, Entity collEntity, Entity memberEntity );
+ EntityManager em, String collName, Entity sourceEntity, Entity targetEntity );
public void visitConnectionEntry(
- EntityCollectionManager ecm, String connType, Entity sourceEntity, Entity targetEntity );
+ EntityManager em, String connType, Entity sourceEntity, Entity targetEntity );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index e780b77..d3f668d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -16,14 +16,12 @@
package org.apache.usergrid.corepersistence;
import java.util.Stack;
-import java.util.logging.Level;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityRef;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -101,127 +99,53 @@ public class CpWalker {
@Override
public void call( Edge edge ) {
+ EntityRef sourceEntityRef = new SimpleEntityRef(
+ edge.getSourceNode().getType(), edge.getSourceNode().getUuid());
+ Entity sourceEntity;
+ try {
+ sourceEntity = em.get( sourceEntityRef );
+ } catch (Exception ex) {
+ logger.error( "Error getting sourceEntity {}:{}, continuing",
+ sourceEntityRef.getType(), sourceEntityRef.getUuid());
+ return;
+ }
+
+ EntityRef targetEntityRef = new SimpleEntityRef(
+ edge.getTargetNode().getType(), edge.getTargetNode().getUuid());
+ Entity targetEntity;
+ try {
+ targetEntity = em.get( targetEntityRef );
+ } catch (Exception ex) {
+ logger.error( "Error getting sourceEntity {}:{}, continuing",
+ sourceEntityRef.getType(), sourceEntityRef.getUuid());
+ return;
+ }
+
if ( CpNamingUtils.isCollectionEdgeType( edge.getType() )) {
String collName = CpNamingUtils.getCollectionName( edgeType );
- String memberType = edge.getTargetNode().getType();
-
- CollectionScope collScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( start.getType()));
- EntityCollectionManager collMgr =
- em.getManagerCache().getEntityCollectionManager(collScope);
-
- org.apache.usergrid.persistence.model.entity.Entity collEntity =
- collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
-
- if (collEntity == null) {
- logger.warn("(Empty collection?) Failed to load collection entity "
- + "{}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getSourceNode().getType(),
- edge.getSourceNode().getUuid(),
- collScope.getApplication(),
- collScope.getOwner(),
- collScope.getName()
- });
- return;
- }
-
- CollectionScope memberScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( memberType ));
- EntityCollectionManager memberMgr =
- em.getManagerCache().getEntityCollectionManager(memberScope);
-
- org.apache.usergrid.persistence.model.entity.Entity memberEntity =
- memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
-
- if (memberEntity == null) {
- logger.warn("(Empty collection?) Failed to load member entity "
- + "{}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid(),
- memberScope.getApplication(),
- memberScope.getOwner(),
- memberScope.getName()
- });
- return;
- }
visitor.visitCollectionEntry(
- memberMgr, collName, collEntity, memberEntity );
+ em, collName, sourceEntity, targetEntity );
// recursion
- if ( !stack.contains( memberEntity.getId() )) {
- stack.push( memberEntity.getId() );
- doWalkCollections( em, memberEntity.getId(), visitor, stack );
+ if ( !stack.contains( targetEntity.getUuid() )) {
+ stack.push( targetEntity.getUuid() );
+ doWalkCollections( em, edge.getSourceNode(), visitor, stack );
stack.pop();
}
+ } else {
- } else if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
-
- String connType = CpNamingUtils.getConnectionType( edgeType );
- String targetEntityType = edge.getTargetNode().getType();
- String sourceEntityType = start.getType();
-
- CollectionScope sourceScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType(sourceEntityType));
- EntityCollectionManager sourceEcm =
- em.getManagerCache().getEntityCollectionManager(sourceScope);
-
- org.apache.usergrid.persistence.model.entity.Entity sourceEntity =
- sourceEcm.load( edge.getSourceNode() ).toBlockingObservable().last();
-
- if (sourceEntity == null) {
- logger.warn("(Empty connection?) Failed to load source entity "
- + "{}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getSourceNode().getType(),
- edge.getSourceNode().getUuid(),
- sourceScope.getApplication(),
- sourceScope.getOwner(),
- sourceScope.getName()
- });
- return;
- }
-
- CollectionScope targetScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType(targetEntityType));
- EntityCollectionManager targetEcm =
- em.getManagerCache().getEntityCollectionManager(targetScope);
-
- org.apache.usergrid.persistence.model.entity.Entity targetEntity =
- targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
-
- if (targetEntity == null) {
- logger.warn("(Empty connection?) Failed to load target entity "
- + "{}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[]{
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid(),
- targetScope.getApplication(),
- targetScope.getOwner(),
- targetScope.getName()
- });
- return;
- }
+ String collName = CpNamingUtils.getConnectionType(edgeType);
visitor.visitConnectionEntry(
- targetEcm, connType, sourceEntity, targetEntity );
+ em, collName, sourceEntity, targetEntity );
// recursion
- if ( !stack.contains( targetEntity.getId() )) {
- stack.push( targetEntity.getId() );
- doWalkCollections( em, targetEntity.getId(), visitor, stack );
+ if ( !stack.contains( targetEntity.getUuid() )) {
+ stack.push( targetEntity.getUuid() );
+ doWalkCollections( em, edge.getTargetNode(), visitor, stack );
stack.pop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index adc0945..54a5dee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -163,14 +163,4 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
factory.rebuildCollectionIndex(appId, collection, po);
}
-
- @Override
- public void repersistAll( ProgressObserver po ) throws Exception {
- factory.repersistAll( po );
- }
-
- @Override
- public void repersistApplication(UUID appId, ProgressObserver po) throws Exception {
- factory.repersistApplication( appId, po );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
index 31018b6..a35263c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
@@ -65,6 +65,7 @@ public class EntityVerifier implements ResultsVerifier {
@Override
public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
ids = ecm.load( idsToLoad ).toBlocking().last();
+ logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
}
@@ -76,7 +77,7 @@ public class EntityVerifier implements ResultsVerifier {
//version wasn't found deindex
if ( savedEntity == null ) {
- logger.warn( "Version for Entity {}:{} not found", entityId.getUuid(), entityId.getUuid() );
+ logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 4973783..757fec2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -189,12 +189,9 @@ public class FilteringLoader implements ResultsLoader {
//now using the scope, load the collection
-
// Get the collection scope and batch load all the versions
- final CollectionScope collScope =
- new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
- scopeName );
-
+ final CollectionScope collScope = new CollectionScopeImpl(
+ applicationScope.getApplication(), ownerId, scopeName );
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 69a3b3f..3684d7e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -694,8 +694,5 @@ public interface EntityManager {
/** For testing purposes */
public void flushManagerCaches();
- public void repersistApplication(
- final UUID appId, final EntityManagerFactory.ProgressObserver po ) throws Exception;
-
public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 714db51..06c3114 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -130,10 +130,6 @@ public interface EntityManagerFactory {
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
- public void repersistAll( ProgressObserver po ) throws Exception;
-
- public void repersistApplication( UUID appId, ProgressObserver po ) throws Exception;
-
public interface ProgressObserver {
public void onProgress( EntityRef source, EntityRef target, String edgeType );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 26e55aa..399bccd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -438,14 +438,4 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
throw new UnsupportedOperationException("Not supported.");
}
-
- @Override
- public void repersistAll( ProgressObserver po ) {
- throw new UnsupportedOperationException("Not supported.");
- }
-
- @Override
- public void repersistApplication(UUID appId, ProgressObserver po) throws Exception {
- throw new UnsupportedOperationException("Not supported yet.");
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index d44dac3..7437c7a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2923,12 +2923,7 @@ public class EntityManagerImpl implements EntityManager {
public void flushManagerCaches() {
// no-op
}
-
- @Override
- public void repersistApplication(UUID appId, EntityManagerFactory.ProgressObserver po) throws Exception {
- throw new UnsupportedOperationException("Not supported.");
- }
-
+
@Override
public void reindex(EntityManagerFactory.ProgressObserver po) throws Exception {
throw new UnsupportedOperationException("Not supported.");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
deleted file mode 100644
index 5b4cccc..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence;
-
-
-import java.util.concurrent.TimeUnit;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.Application;
-import org.apache.usergrid.CoreApplication;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Slf4jReporter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.usergrid.cassandra.Concurrent;
-import org.apache.usergrid.persistence.index.query.Query;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-
-//@RunWith(JukitoRunner.class)
-//@UseModules({ GuiceModule.class })
-@Concurrent()
-public class PerformanceEntityRepersistTest extends AbstractCoreIT {
- private static final Logger logger =
- LoggerFactory.getLogger(PerformanceEntityRepersistTest.class );
-
- private static final MetricRegistry registry = new MetricRegistry();
- private Slf4jReporter reporter;
-
- private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
-
- private static final long writeDelayMs = 100;
- //private static final long readDelayMs = 7;
-
- @Rule
- public Application app = new CoreApplication( setup );
-
-
- @Before
- public void startReporting() {
-
- logger.debug("Starting metrics reporting");
- reporter = Slf4jReporter.forRegistry( registry ).outputTo( logger )
- .convertRatesTo( TimeUnit.SECONDS )
- .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
-
- reporter.start( 10, TimeUnit.SECONDS );
- }
-
-
- @After
- public void printReport() {
-
- logger.debug("Printing metrics report");
- reporter.report();
- reporter.stop();
- }
-
-
- @Test
- public void repersistAll() throws Exception {
-
- logger.info("Started repersistAll()");
-
- final EntityManager em = app.getEntityManager();
-
- // ----------------- create a bunch of entities
-
- final long stopTime = System.currentTimeMillis() + RUNTIME;
-
- Map<String, Object> entityMap = new HashMap<String, Object>() {{
- put("key1", 1000 );
- put("key2", 2000 );
- put("key3", "Some value");
- }};
- Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put("name", "enzo");
- put("color", "orange");
- }};
- Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put("name", "marquee");
- put("color", "grey");
- }};
- Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put("name", "bertha");
- put("color", "tabby");
- }};
-
- Entity cat1 = em.create("cat", cat1map );
- Entity cat2 = em.create("cat", cat2map );
- Entity cat3 = em.create("cat", cat3map );
-
- List<EntityRef> entityRefs = new ArrayList<EntityRef>();
- int entityCount = 0;
- while ( System.currentTimeMillis() < stopTime ) {
-
- final Entity entity;
-
- try {
- entityMap.put("key", entityCount );
- entity = em.create("testType", entityMap );
-
-// em.refreshIndex();
-
- em.createConnection(entity, "herds", cat1);
- em.createConnection(entity, "herds", cat2);
- em.createConnection(entity, "herds", cat3);
-
- } catch (Exception ex) {
- throw new RuntimeException("Error creating entity", ex);
- }
-
- entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( entityCount % 100 == 0 ) {
- logger.info("Created {} entities", entityCount );
- }
-
- entityCount++;
- try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
- }
-
- logger.info("Created {} entities", entityCount);
- em.refreshIndex();
-
- // ----------------- test that we can read them, should work fine
-
- logger.debug("Read the data");
- readData("testTypes", entityCount );
-
- // ----------------- repersist all
-
- logger.debug("Preparing to repersist all");;
-
- final String meterName = this.getClass().getSimpleName() + ".repersist";
- final Meter meter = registry.meter( meterName );
-
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
- int counter = 0;
- @Override
- public void onProgress( EntityRef s, EntityRef t, String etype ) {
- meter.mark();
- logger.debug("Repersisting from {}:{} to {}:{} ", new Object[] {
- s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
- if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
- logger.info("Repersisted {} entities", counter );
- }
- counter++;
- }
- };
-
- try {
- setup.getEmf().repersistAll(po );
-
- registry.remove( meterName );
- logger.info("Repersist complete");
-
- } catch (Exception ex) {
- logger.error("Error repersisting", ex);
- fail();
- }
- em.refreshIndex();
-
- // ----------------- test that we can read them
-
- readData( "testTypes", entityCount );
- }
-
-
-
- private int readData( String collectionName, int expected ) throws Exception {
-
- EntityManager em = app.getEntityManager();
-
- Query q = Query.fromQL("select * where key1=1000");
- q.setLimit(40);
- Results results = em.searchCollection( em.getApplicationRef(), collectionName, q );
-
- int count = 0;
- while ( true ) {
-
- for ( Entity e : results.getEntities() ) {
-
- assertEquals( 2000, e.getProperty("key2"));
-
- Results catResults = em.searchConnectedEntities(e, Query.fromQL("select *"));
- assertEquals( 3, catResults.size() );
-
- if ( count % 100 == 0 ) {
- logger.info( "read {} entities", count);
- }
- count++;
- }
-
- if ( results.hasCursor() ) {
- logger.info( "Counted {} : query again with cursor", count);
- q.setCursor( results.getCursor() );
- results = em.searchCollection( em.getApplicationRef(), collectionName, q );
-
- } else {
- break;
- }
- }
-
- if ( expected != -1 && expected != count ) {
- throw new RuntimeException("Did not get expected "
- + expected + " entities, instead got " + count );
- }
- return count;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index c57f666..9f3c1ea 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -326,23 +326,23 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
- public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
-
- return Observable.create( new Observable.OnSubscribe<VersionSet>() {
-
- @Override
- public void call( final Subscriber<? super VersionSet> subscriber ) {
- try {
- final VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
- UUIDGenerator.newTimeUUID() );
-
- subscriber.onNext( logEntries );
- subscriber.onCompleted();
- }
- catch ( Exception e ) {
- subscriber.onError( e );
- }
- }
- } );
+ public Observable<VersionSet> getLatestVersion(final Collection<Id> entityIds) {
+
+ return Observable.create(new Observable.OnSubscribe<VersionSet>() {
+
+ @Override
+ public void call(final Subscriber<? super VersionSet> subscriber) {
+ try {
+ final VersionSet logEntries = mvccLogEntrySerializationStrategy
+ .load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+
+ subscriber.onNext(logEntries);
+ subscriber.onCompleted();
+
+ } catch (Exception e) {
+ subscriber.onError(e);
+ }
+ }
+ });
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index b1bfe3c..9231a2f 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -199,14 +199,16 @@
<module>core</module>
<module>services</module>
<module>rest</module>
+ <!--
<module>tools</module>
<module>websocket</module>
+ -->
<!--
Re-enable when we have a fix for the AppleJavaExtensions jar issue
https://issues.apache.org/jira/browse/USERGRID-224
<module>launcher</module>
-->
- <module>mongo-emulator</module>
+ <!-- <module>mongo-emulator</module> -->
<!--
Re-enable when query-validator updated to work with Core Persistence.
https://issues.apache.org/jira/browse/USERGRID-221
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index b037b96..ceec656 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -43,6 +43,7 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.persistence.EntityManagerFactory.ProgressObserver;
@Path( "/system" )
@@ -66,7 +67,7 @@ public class SystemResource extends AbstractContextResource {
@GET
@Path( "database/setup" )
public JSONWithPadding getSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
@@ -90,7 +91,7 @@ public class SystemResource extends AbstractContextResource {
@GET
@Path( "superuser/setup" )
public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
@@ -115,14 +116,14 @@ public class SystemResource extends AbstractContextResource {
@PUT
@Path( "index/rebuild" )
public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
ApiResponse response = createApiResponse();
response.setAction( "rebuild indexes" );
- final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ final ProgressObserver po = new ProgressObserver() {
@Override
public void onProgress( EntityRef s, EntityRef t, String etype ) {
logger.info( "Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
@@ -164,8 +165,11 @@ public class SystemResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
@Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ public JSONWithPadding rebuildIndexes(
+ @Context UriInfo ui,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+
throws Exception {
final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
@@ -173,7 +177,7 @@ public class SystemResource extends AbstractContextResource {
response.setAction( "rebuild indexes" );
- final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+ final ProgressObserver po = new ProgressObserver() {
@Override
public void onProgress( EntityRef s, EntityRef t, String etype ) {
logger.info( "Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
@@ -213,10 +217,11 @@ public class SystemResource extends AbstractContextResource {
@RequireSystemAccess
@PUT
@Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ public JSONWithPadding rebuildIndexes(
+ @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
@@ -231,7 +236,8 @@ public class SystemResource extends AbstractContextResource {
}
};
- rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
+ rebuild.setName( String.format(
+ "Index rebuild for app %s and collection %s", appId, collectionName ) );
rebuild.setDaemon( true );
rebuild.start();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java b/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
deleted file mode 100644
index 4c63f61..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-
-
-/**
- * Index rebuild utility for Usergrid. Can be used to rebuild the index for a specific
- * application, a specific application's collection or for an entire Usergrid system.
- */
-public class RepersistAll extends ToolBase {
-
- private static final int PAGE_SIZE = 100;
-
-
- private static final Logger logger = LoggerFactory.getLogger(RepersistAll.class );
-
-
- @Override
- @SuppressWarnings("static-access")
- public Options createOptions() {
-
- Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
- .withDescription( "Cassandra host" ).create( "host" );
-
- Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
- .withDescription( "ElasticSearch host" ).create( "eshost" );
-
- Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
- .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
-
- Options options = new Options();
- options.addOption( hostOpt );
- options.addOption( esHostsOpt );
- options.addOption( esClusterOpt );
-
- return options;
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
- */
- @Override
- public void runTool( CommandLine line ) throws Exception {
- startSpring();
-
- logger.info( "Starting index rebuild" );
-
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
- @Override
- public void onProgress(EntityRef s, EntityRef t, String etype) {
- logger.info("Repersisting from {}:{} to {}:{}", new Object[] {
- s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
- }
- };
-
- emf.rebuildAllIndexes( po );
-
- logger.info( "Finished index rebuild" );
- }
-}
[2/2] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Conflicts:
stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/56d70389
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/56d70389
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/56d70389
Branch: refs/heads/two-dot-o
Commit: 56d703898fd01e5974d2315c0c5dcf143fa4ebc3
Parents: 5a3ece5 884b030
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 10 18:40:46 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 10 18:40:46 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 3 ++-
.../usergrid/corepersistence/CpWalker.java | 5 +++++
.../results/FilteringLoader.java | 13 ++++---------
.../results/VersionVerifier.java | 20 ++++++++++----------
.../MvccEntitySerializationStrategyImpl.java | 2 +-
5 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56d70389/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56d70389/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------