You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/03/19 17:19:27 UTC
[01/50] incubator-usergrid git commit: graphtimers
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-405 5c7a5f86a -> 5e6ed4a12
graphtimers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/56c57018
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/56c57018
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/56c57018
Branch: refs/heads/USERGRID-405
Commit: 56c57018d3b4e8eb391e59bbecffb09fb6c1f1ea
Parents: b3e42dd
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 08:36:23 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 08:36:23 2015 -0600
----------------------------------------------------------------------
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 +++++++++++++++++--
stack/pom.xml | 1 -
3 files changed, 273 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/corepersistence/graph/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/pom.xml b/stack/corepersistence/graph/pom.xml
index e448e74..b4cb45f 100644
--- a/stack/corepersistence/graph/pom.xml
+++ b/stack/corepersistence/graph/pom.xml
@@ -66,13 +66,17 @@
</dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.2</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -88,16 +92,16 @@
<build>
<plugins>
-
+
<!-- <plugin>
<groupId>org.safehaus.chop</groupId>
<artifactId>chop-maven-plugin</artifactId>
<version>${chop.version}</version>
-
+
NOTE: you should be putting most of these variables into your settings.xml
as an automatically activated profile.
-
+
<configuration>
<accessKey>${aws.s3.key}</accessKey>
@@ -119,11 +123,11 @@
<runnerCount>6</runnerCount>
<runnerName>${runner.name}</runnerName>
<securityGroupExceptions>
-
+
Add your own IP address as an exception to allow access
but please do this in the settings.xml file .. essentially
all parameters should be in the settings.xml file.
-
+
<param>${myip.address}/32:24981</param>
<param>${myip.address}/32:22</param>
</securityGroupExceptions>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index df10816..53e116d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -25,6 +25,9 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +59,12 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
+import rx.Notification;
import rx.Observable;
import rx.Observer;
import rx.Subscriber;
+import rx.functions.Action0;
+import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -81,6 +87,30 @@ public class GraphManagerImpl implements GraphManager {
private final EdgeDeleteListener edgeDeleteListener;
private final NodeDeleteListener nodeDeleteListener;
+ private final Timer writeEdgeTimer;
+ private final Meter writeEdgeMeter;
+ private final Meter deleteEdgeMeter;
+ private final Timer deleteEdgeTimer;
+ private final Meter deleteNodeMeter;
+ private final Timer deleteNodeTimer;
+ private final Meter loadEdgesFromSourceMeter;
+ private final Timer loadEdgesFromSourceTimer;
+ private final Meter loadEdgesToTargetMeter;
+ private final Timer loadEdgesToTargetTimer;
+ private final Meter loadEdgesVersionsMeter;
+ private final Timer loadEdgesVersionsTimer;
+ private final Meter loadEdgesFromSourceByTypeMeter;
+ private final Timer loadEdgesFromSourceByTypeTimer;
+ private final Meter loadEdgesToTargetByTypeMeter;
+ private final Timer loadEdgesToTargetByTypeTimer;
+ private final Timer getEdgeTypesFromSourceTimer;
+ private final Meter getEdgeTypesFromSourceMeter;
+ private final Timer getIdTypesFromSourceTimer;
+ private final Meter getIdTypesFromSourceMeter;
+ private final Meter getEdgeTypesToTargetMeter;
+ private final Timer getEdgeTypesToTargetTimer;
+ private final Timer getIdTypesToTargetTimer;
+ private final Meter getIdTypesToTargetMeter;
private Observer<Integer> edgeDeleteSubcriber;
private Observer<Integer> nodeDelete;
@@ -96,7 +126,8 @@ public class GraphManagerImpl implements GraphManager {
final GraphFig graphFig,
final EdgeDeleteListener edgeDeleteListener,
final NodeDeleteListener nodeDeleteListener,
- @Assisted final ApplicationScope scope) {
+ @Assisted final ApplicationScope scope,
+ MetricsFactory metricsFactory) {
ValidationUtils.validateApplicationScope( scope );
@@ -117,6 +148,36 @@ public class GraphManagerImpl implements GraphManager {
this.edgeDeleteSubcriber = MetricSubscriber.INSTANCE;
this.nodeDelete = MetricSubscriber.INSTANCE;
+ this.writeEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+ this.writeEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "write.edge.timer");
+ this.deleteEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.edge.meter");
+ this.deleteEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.edge.timer");
+ this.deleteNodeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.node.meter");
+ this.deleteNodeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.node.timer");
+ this.loadEdgesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.meter");
+ this.loadEdgesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.timer");
+ this.loadEdgesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.meter");
+ this.loadEdgesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.timer");
+ this.loadEdgesVersionsMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.versions.meter");
+ this.loadEdgesVersionsTimer = metricsFactory.getTimer(GraphManagerImpl.class,"load.versions.timer");
+ this.loadEdgesFromSourceByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.type.meter");
+ this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.type.timer");
+ this.loadEdgesToTargetByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.type.meter");
+ this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.type.timer");
+
+ this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.from.timer");
+ this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+
+ this.getIdTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.idtype.from.timer");
+ this.getIdTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.from.meter");
+
+ this.getEdgeTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.to.timer");
+ this.getEdgeTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.to.meter");
+
+ this.getIdTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "get.idtype.to.timer");
+ this.getIdTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.to.meter");
+
+
}
@@ -125,6 +186,8 @@ public class GraphManagerImpl implements GraphManager {
GraphValidation.validateEdge( edge );
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false );
+ final Timer.Context timer = writeEdgeTimer.time();
+ final Meter meter = writeEdgeMeter;
return Observable.from( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
@Override
@@ -143,45 +206,72 @@ public class GraphManagerImpl implements GraphManager {
return edge;
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> deleteEdge( final Edge edge ) {
- GraphValidation.validateEdge( edge );
+ GraphValidation.validateEdge(edge);
- final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
+ final MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
-
- return Observable.from( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
+ final Timer.Context timer = deleteEdgeTimer.time();
+ final Meter meter = deleteEdgeMeter;
+ return Observable.from(markedEdge).map(new Func1<MarkedEdge, Edge>() {
@Override
- public Edge call( final MarkedEdge edge ) {
+ public Edge call(final MarkedEdge edge) {
final UUID timestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge, timestamp );
+ final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge(scope, edge, timestamp);
- LOG.debug( "Marking edge {} as deleted to commit log", edge );
- HystrixCassandra.user( edgeMutation );
+ LOG.debug("Marking edge {} as deleted to commit log", edge);
+ HystrixCassandra.user(edgeMutation);
//HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
// timestamp )).subscribeOn( Schedulers.io() ).subscribe( edgeDeleteSubcriber );
- edgeDeleteListener.receive( scope, markedEdge, timestamp ).subscribeOn( Schedulers.io() )
- .subscribe( edgeDeleteSubcriber );
+ edgeDeleteListener.receive(scope, markedEdge, timestamp).subscribeOn(Schedulers.io())
+ .subscribe(edgeDeleteSubcriber);
return edge;
}
- } );
+ })
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Id> deleteNode( final Id node, final long timestamp ) {
+ final Timer.Context timer = deleteNodeTimer.time();
+ final Meter meter = deleteNodeMeter;
return Observable.from( node ).map( new Func1<Id, Id>() {
@Override
public Id call( final Id id ) {
@@ -205,114 +295,250 @@ public class GraphManagerImpl implements GraphManager {
return id;
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super Id>>() {
+ @Override
+ public void call(Notification<? super Id> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgeVersions( final SearchByEdge searchByEdge ) {
+ final Timer.Context timer = loadEdgesVersionsTimer.time();
+ final Meter meter = loadEdgesVersionsMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(searchByEdge.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
+ final Timer.Context timer = loadEdgesFromSourceTimer.time();
+ final Meter meter = loadEdgesFromSourceMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
+ final Timer.Context timer = loadEdgesToTargetTimer.time();
+ final Meter meter = loadEdgesToTargetMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
+ final Timer.Context timer = loadEdgesFromSourceByTypeTimer.time();
+ final Meter meter = loadEdgesFromSourceByTypeMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
-
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
+ final Timer.Context timer = loadEdgesToTargetByTypeTimer.time();
+ final Meter meter = loadEdgesToTargetByTypeMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
-
+ final Timer.Context timer = getEdgeTypesFromSourceTimer.time();
+ final Meter meter = getEdgeTypesFromSourceMeter;
return Observable.create( new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
+ final Timer.Context timer = getIdTypesFromSourceTimer.time();
+ final Meter meter = getIdTypesFromSourceMeter;
return Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
-
+ final Timer.Context timer = getEdgeTypesToTargetTimer.time();
+ final Meter meter = getEdgeTypesToTargetMeter;
return Observable.create( new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
+ final Timer.Context timer = getIdTypesToTargetTimer.time();
+ final Meter meter = getIdTypesToTargetMeter;
return Observable.create( new ObservableIterator<String>( "getIdTypesToTarget" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 25b2258..73f2cbd 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,7 +109,6 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
[40/50] incubator-usergrid git commit: Merge branch 'USERGRID-466'
into USERGRID-466-change-write-consistency
Posted by to...@apache.org.
Merge branch 'USERGRID-466' into USERGRID-466-change-write-consistency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/45271d7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/45271d7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/45271d7a
Branch: refs/heads/USERGRID-405
Commit: 45271d7a26ad61063a13307ad567882afe387acc
Parents: 8cea08d 9d00c22
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:24:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:24:16 2015 -0600
----------------------------------------------------------------------
.../main/groovy/configure_elasticsearch.groovy | 1 -
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 31 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
12 files changed, 232 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
[03/50] incubator-usergrid git commit: add timer and meter to
collection
Posted by to...@apache.org.
add timer and meter to collection
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8ccc3ec5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8ccc3ec5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8ccc3ec5
Branch: refs/heads/USERGRID-405
Commit: 8ccc3ec5cf9a29381f2acdb63331f9de447dfd64
Parents: 06fe609
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 09:04:41 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 09:04:41 2015 -0600
----------------------------------------------------------------------
.../EntityCollectionManagerFactoryImpl.java | 8 +-
.../impl/EntityCollectionManagerImpl.java | 120 ++++++++++++++-----
2 files changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ccc3ec5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 4e04c2e..bb1b56a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import java.util.concurrent.ExecutionException;
@@ -80,6 +81,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final EntityDeletedFactory entityDeletedFactory;
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
+ private final MetricsFactory metricsFactory;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
@@ -91,7 +93,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
mvccLogEntrySerializationStrategy, keyspace, entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope );
+ entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -117,7 +119,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final EntityVersionCreatedFactory entityVersionCreatedFactory,
final EntityDeletedFactory entityDeletedFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
- final EntityCacheFig entityCacheFig) {
+ final EntityCacheFig entityCacheFig,
+ MetricsFactory metricsFactory) {
this.writeStart = writeStart;
this.writeUpdate = writeUpdate;
@@ -136,6 +139,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.entityDeletedFactory = entityDeletedFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
+ this.metricsFactory = metricsFactory;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ccc3ec5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 7c467c6..391e9a5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,6 +23,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,8 +75,10 @@ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
+import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -113,6 +118,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final TaskExecutor taskExecutor;
private final Keyspace keyspace;
+ private final Timer writeTimer;
+ private final Meter writeMeter;
+ private final Timer deleteTimer;
+ private final Timer updateTimer;
+ private final Timer loadTimer;
+ private final Timer getLatestTimer;
@Inject
@@ -133,7 +144,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final EntityVersionCreatedFactory entityVersionCreatedFactory,
final EntityDeletedFactory entityDeletedFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
+ @Assisted final CollectionScope collectionScope,
+ final MetricsFactory metricsFactory
+
) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
@@ -160,6 +173,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.collectionScope = collectionScope;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
+ this.writeTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"write.timer");
+ this.writeMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "write.meter");
+ this.deleteTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "delete.timer");
+ this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"update.timer");
+ this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"load.timer");
+ this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"latest.timer");
}
@@ -184,6 +203,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
+ final Timer.Context timer = writeTimer.time();
return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
public void call(final Entity entity) {
@@ -192,7 +212,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
//post-processing to come later. leave it empty for now.
}
- }).doOnError(rollback);
+ }).doOnError(rollback)
+ .doOnEach(new Action1<Notification<? super Entity>>() {
+ @Override
+ public void call(Notification<? super Entity> notification) {
+ writeMeter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -203,21 +235,27 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
+ final Timer.Context timer = deleteTimer.time();
Observable<Id> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId))
- .map( markStart)
- .doOnNext( markCommit)
- .map( new Func1<CollectionIoEvent<MvccEntity>, Id>() {
-
+ .map(markStart)
+ .doOnNext(markCommit)
+ .map(new Func1<CollectionIoEvent<MvccEntity>, Id>() {
+
+ @Override
+ public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
+ MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
+ Task<Void> task = entityDeletedFactory
+ .getTask(collectionScope, entity.getId(), entity.getVersion());
+ taskExecutor.submit(task);
+ return entity.getId();
+ }
+ }
+ ) .doOnCompleted(new Action0() {
@Override
- public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityDeletedFactory
- .getTask( collectionScope, entity.getId(), entity.getVersion());
- taskExecutor.submit(task);
- return entity.getId();
+ public void call() {
+ timer.stop();
}
- }
- );
+ });;
return o;
}
@@ -230,18 +268,25 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return load( Collections.singleton( entityId ) ).flatMap( new Func1<EntitySet, Observable<Entity>>() {
+ final Timer.Context timer = loadTimer.time();
+ return load( Collections.singleton( entityId ) ).flatMap(new Func1<EntitySet, Observable<Entity>>() {
@Override
- public Observable<Entity> call( final EntitySet entitySet ) {
- final MvccEntity entity = entitySet.getEntity( entityId );
+ public Observable<Entity> call(final EntitySet entitySet) {
+ final MvccEntity entity = entitySet.getEntity(entityId);
- if ( entity == null || !entity.getEntity().isPresent() ) {
+ if (entity == null || !entity.getEntity().isPresent()) {
return Observable.empty();
}
- return Observable.from( entity.getEntity().get() );
+ return Observable.from(entity.getEntity().get());
}
- } );
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -253,6 +298,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
+ final Timer.Context timer = loadTimer.time();
return Observable.create( new Observable.OnSubscribe<EntitySet>() {
@@ -269,7 +315,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } );
+ } )
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -314,13 +366,14 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
- return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
+ final Timer.Context timer = updateTimer.time();
+ return observable.map( writeCommit ).doOnNext(new Action1<Entity>() {
@Override
- public void call( final Entity entity ) {
- logger.debug( "sending entity to the queue" );
+ public void call(final Entity entity) {
+ logger.debug("sending entity to the queue");
//we an update, signal the fix
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
+ taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope, entity));
//TODO T.N Change this to fire a task
// Observable.from( new CollectionIoEvent<Id>(collectionScope,
@@ -328,7 +381,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
- } ).doOnError( rollback );
+ }).doOnError(rollback)
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -362,6 +421,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
+ final Timer.Context timer = getLatestTimer.time();
return Observable.create( new Observable.OnSubscribe<VersionSet>() {
@Override
@@ -377,7 +437,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } );
+ } )
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
[41/50] incubator-usergrid git commit: Updated futures impl for
different queues
Posted by to...@apache.org.
Updated futures impl for different queues
Added onError to catch errors
Added a gauge so we can track index operation in flight
Updated queue scope to only be a name, since we only use them at the system level.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2d6ae369
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2d6ae369
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2d6ae369
Branch: refs/heads/USERGRID-405
Commit: 2d6ae3698e2faff733e73d17aff569a3850a6485
Parents: 361060e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 13 16:19:40 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 13 16:19:40 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 +-
.../corepersistence/StaleIndexCleanupTest.java | 4 +
.../persistence/core/future/BetterFuture.java | 43 ++++--
.../core/metrics/MetricsFactory.java | 9 ++
.../core/metrics/MetricsFactoryImpl.java | 121 ++++++++++------
stack/corepersistence/queryindex/pom.xml | 8 +-
.../persistence/index/IndexBufferConsumer.java | 11 ++
.../usergrid/persistence/index/IndexFig.java | 11 ++
.../index/IndexOperationMessage.java | 7 +-
.../persistence/index/guice/IndexModule.java | 11 +-
.../persistence/index/impl/BufferQueue.java | 22 ++-
.../index/impl/BufferQueueInMemoryImpl.java | 53 ++++---
.../index/impl/BufferQueueSQSImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 142 ++-----------------
.../index/impl/EsIndexBufferConsumerImpl.java | 131 ++++++++++-------
.../index/guice/TestIndexModule.java | 8 +-
.../index/impl/BufferQueueSQSImplTest.java | 5 +
.../impl/EntityConnectionIndexImplTest.java | 3 -
.../persistence/index/impl/EntityIndexTest.java | 1 -
.../persistence/index/impl/EsTestUtils.java | 48 -------
.../cassandra/ManagementServiceImpl.java | 9 +-
.../notifications/NotificationsService.java | 50 ++++---
22 files changed, 359 insertions(+), 353 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 3230faa..161037b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -71,12 +71,7 @@ public class CoreModule extends AbstractModule {
install( new CommonModule());
install(new CollectionModule());
install(new GraphModule());
- install( new IndexModule() {
- @Override
- public void wireBufferQueue() {
- bind(BufferQueue.class).to( BufferQueueSQSImpl.class );
- }
- } );
+ install( new IndexModule() );
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
// install(new QueueModule());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 82d8f93..8d31b69 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -310,6 +310,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
em.delete( thing );
}
+
+ //put this into the top of the queue, once it's acked we've been flushed
+ em.refreshIndex();
+
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
index 201fa9a..777ac52 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
@@ -16,28 +16,53 @@
*/
package org.apache.usergrid.persistence.core.future;
+
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
+
/**
* Future without the exception nastiness
*/
-public class BetterFuture<T> extends FutureTask<T> {
- public BetterFuture(Callable<T> callable){
- super(callable);
+public class BetterFuture<T> extends FutureTask<T> {
+
+ private Throwable error;
+
+
+ public BetterFuture( Callable<T> callable ) {
+ super( callable );
}
- public void done(){
+
+ public void setError( final Throwable t ) {
+ this.error = t;
+ }
+
+
+ public void done() {
run();
}
- public T get(){
+
+ public T get() {
+
+ T returnValue = null;
+
try {
- return super.get();
- }catch (Exception e){
- throw new RuntimeException(e);
+ returnValue = super.get();
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+ catch ( ExecutionException e ) {
+ //swallow
}
- }
+ if ( error != null ) {
+ throw new RuntimeException( "Error in getting future", error );
+ }
+ return returnValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
index 453e556..62a5cb9 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
@@ -31,4 +31,13 @@ public interface MetricsFactory {
Counter getCounter(Class<?> klass, String name);
Meter getMeter(Class<?> klass, String name);
+
+ /**
+ * Get a gauge and create it
+ * @param clazz
+ * @param name
+ * @param gauge
+ * @return
+ */
+ void addGauge( Class<?> clazz, String name, Gauge<?> gauge );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 6d0881b..904e56a 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -17,17 +17,27 @@
package org.apache.usergrid.persistence.core.metrics;
-import com.codahale.metrics.*;
+import java.net.InetSocketAddress;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.net.InetSocketAddress;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
/**
* Singleton class to manage metrics.
@@ -39,74 +49,101 @@ public class MetricsFactoryImpl implements MetricsFactory {
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
- private ConcurrentHashMap<String,Metric> hashMap;
- private static final Logger LOG = LoggerFactory.getLogger(MetricsFactoryImpl.class);
+ private ConcurrentHashMap<String, Metric> hashMap;
+ private static final Logger LOG = LoggerFactory.getLogger( MetricsFactoryImpl.class );
+
@Inject
- public MetricsFactoryImpl(MetricsFig metricsFig) {
+ public MetricsFactoryImpl( MetricsFig metricsFig ) {
registry = new MetricRegistry();
String metricsHost = metricsFig.getHost();
- if(!metricsHost.equals("false")) {
- Graphite graphite = new Graphite(new InetSocketAddress(metricsHost, 2003));
- graphiteReporter = GraphiteReporter.forRegistry(registry)
- .prefixedWith("usergrid-metrics")
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .filter(MetricFilter.ALL)
- .build(graphite);
- graphiteReporter.start(30, TimeUnit.SECONDS);
- }else {
- LOG.warn("MetricsService:Logger not started.");
+ if ( !metricsHost.equals( "false" ) ) {
+ Graphite graphite = new Graphite( new InetSocketAddress( metricsHost, 2003 ) );
+ graphiteReporter = GraphiteReporter.forRegistry( registry ).prefixedWith( "usergrid-metrics" )
+ .convertRatesTo( TimeUnit.SECONDS )
+ .convertDurationsTo( TimeUnit.MILLISECONDS ).filter( MetricFilter.ALL )
+ .build( graphite );
+ graphiteReporter.start( 30, TimeUnit.SECONDS );
+ }
+ else {
+ LOG.warn( "MetricsService:Logger not started." );
}
hashMap = new ConcurrentHashMap<String, Metric>();
- jmxReporter = JmxReporter.forRegistry(registry).build();
+ jmxReporter = JmxReporter.forRegistry( registry ).build();
jmxReporter.start();
}
+
@Override
public MetricRegistry getRegistry() {
return registry;
}
+
@Override
- public Timer getTimer(Class<?> klass, String name) {
- return getMetric(Timer.class, klass, name);
+ public Timer getTimer( Class<?> klass, String name ) {
+ return getMetric( Timer.class, klass, name );
}
+
@Override
- public Histogram getHistogram(Class<?> klass, String name) {
- return getMetric(Histogram.class, klass, name);
+ public Histogram getHistogram( Class<?> klass, String name ) {
+ return getMetric( Histogram.class, klass, name );
}
+
@Override
- public Counter getCounter(Class<?> klass, String name) {
- return getMetric(Counter.class, klass, name);
+ public Counter getCounter( Class<?> klass, String name ) {
+ return getMetric( Counter.class, klass, name );
}
+
@Override
- public Meter getMeter(Class<?> klass, String name) {
- return getMetric(Meter.class, klass, name);
+ public Meter getMeter( Class<?> klass, String name ) {
+ return getMetric( Meter.class, klass, name );
}
- private <T> T getMetric(Class<T> metricClass, Class<?> klass, String name) {
+
+ @Override
+ public void addGauge( final Class<?> clazz, final String name, final Gauge<?> gauge ) {
+
+ this.getRegistry().register( MetricRegistry.name( clazz, name ), gauge );
+ }
+
+
+ private <T> T getMetric( Class<T> metricClass, Class<?> klass, String name ) {
String key = metricClass.getName() + klass.getName() + name;
- Metric metric = hashMap.get(key);
- if (metric == null) {
- if (metricClass == Histogram.class) {
- metric = this.getRegistry().histogram(MetricRegistry.name(klass, name));
+ Metric metric = hashMap.get( key );
+ if ( metric == null ) {
+ if ( metricClass == Histogram.class ) {
+ metric = this.getRegistry().histogram( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Timer.class) {
- metric = this.getRegistry().timer(MetricRegistry.name(klass, name));
+ if ( metricClass == Timer.class ) {
+ metric = this.getRegistry().timer( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Meter.class) {
- metric = this.getRegistry().meter(MetricRegistry.name(klass, name));
+ if ( metricClass == Meter.class ) {
+ metric = this.getRegistry().meter( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Counter.class) {
- metric = this.getRegistry().counter(MetricRegistry.name(klass, name));
+ if ( metricClass == Counter.class ) {
+ metric = this.getRegistry().counter( MetricRegistry.name( klass, name ) );
}
- hashMap.put(key, metric);
+
+
+ hashMap.put( key, metric );
}
- return (T) metric;
+ return ( T ) metric;
+ }
+
+
+ /**
+ *
+ * @param metricClass
+ * @param klass
+ * @param name
+ * @return
+ */
+ private String getKey( Class<?> metricClass, Class<?> klass, String name ) {
+ return metricClass.getName() + klass.getName() + name;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index af843ad..5f01ee7 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -139,7 +139,13 @@
<classifier>tests</classifier>
</dependency>
-
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
index ac7489c..40c7852 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
@@ -23,4 +23,15 @@ package org.apache.usergrid.persistence.index;
* Classy class class.
*/
public interface IndexBufferConsumer {
+
+
+ /**
+ * Start the consumer
+ */
+ public void start();
+
+ /**
+ * Stop the consumers
+ */
+ public void stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 445789f..6be8234 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -24,6 +24,8 @@ import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
import org.safehaus.guicyfig.Key;
+import org.apache.usergrid.persistence.index.guice.QueueProvider;
+
@FigSingleton
public interface IndexFig extends GuicyFig {
@@ -86,6 +88,11 @@ public interface IndexFig extends GuicyFig {
*/
public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -190,4 +197,8 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
+ @Default( "LOCAL" )
+ @Key( ELASTICSEARCH_QUEUE_IMPL )
+ String getQueueImplementation();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index 33b68cd..5686e26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -128,6 +128,11 @@ public class IndexOperationMessage implements Serializable {
}
public void done() {
- getFuture().done();
+ //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
+ final BetterFuture<IndexOperationMessage> future = getFuture();
+
+ if(future != null ){
+ future.done();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index b03e1c0..95f3bd4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -23,6 +23,7 @@ import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
@@ -33,7 +34,7 @@ import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.safehaus.guicyfig.GuicyFigModule;
-public abstract class IndexModule extends AbstractModule {
+public class IndexModule extends AbstractModule {
@Override
protected void configure() {
@@ -50,14 +51,10 @@ public abstract class IndexModule extends AbstractModule {
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
- wireBufferQueue();
- }
+ bind( BufferQueue.class).toProvider( QueueProvider.class );
+ }
- /**
- * Write the <class>BufferQueue</class> for this implementation
- */
- public abstract void wireBufferQueue();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
index ffc3b90..76b49c2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
@@ -32,25 +32,37 @@ import org.apache.usergrid.persistence.index.IndexOperationMessage;
public interface BufferQueue {
/**
- * Offer the indexoperation message
+ * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
+ * Other queues may return the future after ack on the offer. See the implementation documentation for details.
* @param operation
*/
public void offer(final IndexOperationMessage operation);
/**
- * Perform a take, potentially blocking. Until takesize is available, or timeout has elapsed
+ * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
+ * May return less than the take size, but will never return null
+ *
* @param takeSize
* @param timeout
* @param timeUnit
- * @return
+ * @return A null safe lid
*/
public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
/**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented
+ * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
+ * This will set the future as done in in memory operations
+ *
* @param messages
*/
- public void ack(List<IndexOperationMessage> messages);
+ public void ack(final List<IndexOperationMessage> messages);
+
+ /**
+ * Mark these message as failed. Set the exception in the future on local operation
+ *
+ * @param messages
+ */
+ public void fail(final List<IndexOperationMessage> messages, final Throwable t);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index 6716fd1..998c086 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
@@ -47,7 +48,6 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
messages.offer( operation );
- operation.done();
}
@@ -55,30 +55,30 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
+ try {
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
- //loop until we're we're full or we time out
- do {
- try {
+ messages.drainTo( response, takeSize );
- final long remaining = endTime - System.currentTimeMillis();
+ //we got something, go process it
+ if ( response.size() > 0 ) {
+ return response;
+ }
- //we received 1, try to drain
- IndexOperationMessage polled = messages.poll( remaining, timeUnit );
- //drain
- if ( polled != null ) {
- response.add( polled );
- messages.drainTo( response, takeSize - response.size() );
- }
- }
- catch ( InterruptedException ie ) {
- //swallow
+ final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+ if ( polled != null ) {
+ response.add( polled );
+
+ //try to add more
+ messages.drainTo( response, takeSize - 1 );
}
}
- while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+
return response;
}
@@ -86,6 +86,23 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void ack( final List<IndexOperationMessage> messages ) {
- //no op for this
+ //if we have a future ack it
+ for ( final IndexOperationMessage op : messages ) {
+ op.done();
+ }
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+
+
+ for ( final IndexOperationMessage op : messages ) {
+ final BetterFuture<IndexOperationMessage> future = op.getFuture();
+
+ if ( future != null ) {
+ future.setError( t );
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 4a07704..b8d162b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -101,7 +101,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig,
final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
final QueueScope queueScope =
- new QueueScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), QUEUE_NAME );
+ new QueueScopeImpl( QUEUE_NAME );
this.queue = queueManagerFactory.getQueueManager( queueScope );
this.indexFig = indexFig;
@@ -260,6 +260,12 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+ //no op, just let it retry after the queue timeout
+ }
+
+
/** Read the object from Base64 string. */
private IndexOperationMessage fromString( String s ) throws IOException {
IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fa50734..8be044f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -47,6 +47,8 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
@@ -111,8 +113,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer addWriteAliasTimer;
private final Timer addReadAliasTimer;
private final Timer searchTimer;
- private final Timer allVersionsTimerFuture;
- private final Timer deletePreviousTimerFuture;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -127,7 +127,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexFig config;
- private final MetricsFactory metricsFactory;
//number of times to wait for the index to refresh properly.
@@ -148,8 +147,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private Timer refreshTimer;
private Timer cursorTimer;
private Timer getVersionsTimer;
- private Timer allVersionsTimer;
- private Timer deletePreviousTimer;
private final MapManager mapManager;
@@ -168,11 +165,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier( config, appScope );
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
- this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
.getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
this.removeAliasTimer = metricsFactory
@@ -191,14 +187,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
this.getVersionsTimer =metricsFactory
.getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
- this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
- this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
- this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer.future" );
- this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer.future" );
+
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
@@ -394,8 +383,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
*/
private void createMappings(final String indexName) throws IOException {
- XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), DEFAULT_TYPE );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(),
+ DEFAULT_TYPE );
//Added For Graphite Metrics
@@ -421,7 +410,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName(indexScope);
+ final String context = IndexingUtils.createContextName( indexScope );
final String[] entityTypes = searchTypes.getTypeNames();
QueryBuilder qb = query.createQueryBuilder(context);
@@ -632,7 +621,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public int getPendingTasks() {
final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin()
- .cluster().pendingClusterTasks(new PendingClusterTasksRequest()).actionGet();
+ .cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
return tasksResponse.pendingTasks().size();
}
@@ -674,114 +663,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
-// @Override
-// public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
-// String idString = IndexingUtils.idString(entityId).toLowerCase();
-//
-// final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
-//
-// //Added For Graphite Metrics
-// final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
-// final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
-//
-// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
-// .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
-//
-// response.addListener( new ActionListener<DeleteByQueryResponse>() {
-//
-// @Override
-// public void onResponse( DeleteByQueryResponse response) {
-// timeDeleteAllVersions.stop();
-// logger
-// .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
-// entityId.getUuid(), response.status().toString() );
-//
-// checkDeleteByQueryResponse(tqb, response);
-// }
-//
-//
-// @Override
-// public void onFailure( Throwable e ) {
-// timeDeleteAllVersions.stop();
-// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
-// entityId.getUuid(), e);
-//
-//
-// }
-// });
-// timeDeleteAllVersionsFuture.stop();
-// return response;
-// }
-//
-//
-// @Override
-// public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
-//
-// String idString = IndexingUtils.idString( entityId ).toLowerCase();
-//
-// final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
-// QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
-// FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
-// );
-//
-// //Added For Graphite Metrics
-// //Checks the time from the execute to the response below
-// final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
-// final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
-// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
-// .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
-//
-// //Added For Graphite Metrics
-// response.addListener(new ActionListener<DeleteByQueryResponse>() {
-// @Override
-// public void onResponse(DeleteByQueryResponse response) {
-// timeDeletePreviousVersions.stop();
-// //error message needs to be retooled so that it describes the entity more throughly
-// logger
-// .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
-// entityId.getType(), entityId.getUuid(), version, response.status().toString());
-//
-// checkDeleteByQueryResponse( fqb, response );
-// }
-//
-//
-// @Override
-// public void onFailure( Throwable e ) {
-// timeDeletePreviousVersions.stop();
-// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
-// entityId.getUuid(), e );
-// }
-// } );
-//
-// timeDeletePreviousVersionFuture.stop();
-//
-// return response;
-// }
-
-
- /**
- * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
- */
- private void checkDeleteByQueryResponse(
- final QueryBuilder query, final DeleteByQueryResponse response ) {
-
- for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
- final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
-
- for ( ShardOperationFailedException failedException : failures ) {
- logger.error( String.format("Unable to delete by query %s. "
- + "Failed with code %d and reason %s on shard %s in index %s",
- query.toString(),
- failedException.status().getStatus(),
- failedException.reason(),
- failedException.shardId(),
- failedException.index() )
- );
- }
-
- }
- }
-
/**
* Completely delete an index.
@@ -856,8 +737,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public Health getIndexHealth() {
try {
- ClusterHealthResponse chr = esProvider.getClient().admin().cluster().health(
- new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)})).get();
+ final ActionFuture<ClusterHealthResponse> future = esProvider.getClient().admin().cluster().health(
+ new ClusterHealthRequest( new String[] { indexIdentifier.getIndex( null ) } ) );
+
+ //only wait 2 seconds max
+ ClusterHealthResponse chr = future.actionGet( 2000 );
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 836ec3d..5259a26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -20,31 +20,29 @@
package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexBufferConsumer;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
-import org.elasticsearch.action.ActionRequestBuilder;
+
import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
import rx.functions.Action1;
-import rx.functions.Action2;
import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
@@ -52,7 +50,6 @@ import rx.schedulers.Schedulers;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -69,6 +66,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Timer flushTimer;
private final Counter indexSizeCounter;
+ private final Counter indexErrorCounter;
private final Meter flushMeter;
private final Timer produceTimer;
private final BufferQueue bufferQueue;
@@ -80,13 +78,28 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private Object mutex = new Object();
+
+ private AtomicLong inFlight = new AtomicLong( );
+
@Inject
public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
- this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
- this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
+ this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "buffer.flush");
+ this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "buffer.meter");
+ this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "buffer.size");
+ this.indexErrorCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "error.count");
+
+ //wire up the gauge of inflight messages
+ metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return inFlight.longValue();
+ }
+ } );
+
+
+
this.config = config;
this.failureMonitor = new FailureMonitorImpl(config,provider);
this.client = provider.getClient();
@@ -130,66 +143,67 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private void startWorker(){
synchronized ( mutex) {
- final AtomicInteger countFail = new AtomicInteger();
+ Observable<List<IndexOperationMessage>> consumer = Observable.create(
+ new Observable.OnSubscribe<List<IndexOperationMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
- Observable<List<IndexOperationMessage>> consumer = Observable.create( new Observable.OnSubscribe<List<IndexOperationMessage>>() {
- @Override
- public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
+ //name our thread so it's easy to see
+ Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
- //name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
+ List<IndexOperationMessage> drainList;
+ do {
+ try {
- List<IndexOperationMessage> drainList;
- do {
- try {
+ Timer.Context timer = produceTimer.time();
- Timer.Context timer = produceTimer.time();
+ drainList = bufferQueue
+ .take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
+ TimeUnit.MILLISECONDS );
- drainList = bufferQueue.take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
- TimeUnit.MILLISECONDS );
+ subscriber.onNext( drainList );
- subscriber.onNext( drainList );
+ //take since we're in flight
+ inFlight.addAndGet( drainList.size() );
- timer.stop();
+ timer.stop();
+ }
- countFail.set( 0 );
- }
- catch ( EsRejectedExecutionException err ) {
- countFail.incrementAndGet();
- log.error(
- "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. " + "Failed {} consecutive times", config.getFailRefreshCount(),
- countFail.get() );
+ catch ( Exception e ) {
+ final long sleepTime = config.getFailureRetryTime();
- //es rejected the exception, sleep and retry in the queue
- try {
- Thread.sleep( config.getFailureRetryTime() );
- }
- catch ( InterruptedException e ) {
- //swallow
- }
- }
- catch ( Exception e ) {
- int count = countFail.incrementAndGet();
- log.error( "failed to dequeue", e );
- if ( count > 200 ) {
- log.error( "Shutting down index drain due to repetitive failures" );
+ log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, e );
+
+ try {
+ Thread.sleep( sleepTime );
+ }
+ catch ( InterruptedException ie ) {
+ //swallow
+ }
+
+ indexErrorCounter.inc();
}
}
+ while ( true );
}
- while ( true );
- }
- } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
+ } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
public void call( List<IndexOperationMessage> containerList ) {
- if ( containerList.size() > 0 ) {
- flushMeter.mark( containerList.size() );
- Timer.Context time = flushTimer.time();
- execute( containerList );
- time.stop();
+ if ( containerList.size() == 0 ) {
+ return;
}
+
+ flushMeter.mark( containerList.size() );
+ Timer.Context time = flushTimer.time();
+
+
+ execute( containerList );
+
+ time.stop();
+
}
} )
//ack after we process
@@ -197,6 +211,16 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
@Override
public void call( final List<IndexOperationMessage> indexOperationMessages ) {
bufferQueue.ack( indexOperationMessages );
+ //release so we know we've done processing
+ inFlight.addAndGet( -1 * indexOperationMessages.size() );
+ }
+ } ).doOnError( new Action1<Throwable>() {
+ @Override
+ public void call( final Throwable throwable ) {
+
+ log.error( "An exception occurred when trying to deque and write to elasticsearch. Ignoring",
+ throwable );
+ indexErrorCounter.inc();
}
} );
@@ -236,7 +260,8 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//collection all the operations into a single stream
.reduce( initRequest(), new Func2<BulkRequestBuilder, BatchRequest, BulkRequestBuilder>() {
@Override
- public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder,
+ final BatchRequest batchRequest ) {
batchRequest.doOperation( client, bulkRequestBuilder );
return bulkRequestBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 57c2fab..50b994d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -35,12 +35,6 @@ public class TestIndexModule extends TestModule {
// configure collections and our core astyanax framework
install( new CollectionModule() );
- install( new IndexModule() {
- @Override
- public void wireBufferQueue() {
- bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
-// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
- }
- } );
+ install( new IndexModule() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index 6922c15..9a362cb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
@@ -59,6 +60,10 @@ public class BufferQueueSQSImplTest {
@Rule
public MigrationManagerRule migrationManagerRule;
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
@Inject
public QueueManagerFactory queueManagerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index c5f3488..a399809 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -140,7 +140,6 @@ public class EntityConnectionIndexImplTest extends BaseIT {
personLikesIndex.refresh();
- EsTestUtils.waitForTasks(personLikesIndex);
Thread.sleep( 2000 );
// now, let's search for muffins
@@ -270,8 +269,6 @@ public class EntityConnectionIndexImplTest extends BaseIT {
batch.execute().get();
personLikesIndex.refresh();
- EsTestUtils.waitForTasks( personLikesIndex );
- Thread.sleep( 2000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex.search( searchScope,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a2135a3..ca9bf79 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -94,7 +94,6 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
- Thread.sleep(100000000);
testQueries( indexScope, searchTypes, entityIndex );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
deleted file mode 100644
index 30f0ed0..0000000
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import org.apache.usergrid.persistence.index.EntityIndex;
-
-
-/**
- * Utilities to make testing ES easier
- */
-public class EsTestUtils {
-
-
- /**
- * Checks to see if we have pending tasks in the cluster. If so waits until they are finished. Adding
- * new types can cause lag even after refresh since the type mapping needs applied
- * @param index
- */
- public static void waitForTasks(final EntityIndex index){
-
- while(index.getPendingTasks() > 0){
- try {
- Thread.sleep( 100 );
- }
- catch ( InterruptedException e ) {
- //swallow
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 35ed091..854c3e0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -540,7 +540,6 @@ public class ManagementServiceImpl implements ManagementService {
em.addToCollection( organizationEntity, "users", new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ) );
- em.refreshIndex();
writeUserToken( smf.getManagementAppId(), organizationEntity, encryptionService
.plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.ORGANIZATION ), user.getUuid(),
@@ -557,7 +556,7 @@ public class ManagementServiceImpl implements ManagementService {
startOrganizationActivationFlow( organization );
- em.refreshIndex();
+
return organization;
}
@@ -1649,7 +1648,7 @@ public class ManagementServiceImpl implements ManagementService {
properties.put( "appUuid", applicationId );
Entity appInfo = em.create( applicationId, APPLICATION_INFO, properties );
- em.refreshIndex();
+
writeUserToken( smf.getManagementAppId(), appInfo, encryptionService
.plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
@@ -1670,7 +1669,7 @@ public class ManagementServiceImpl implements ManagementService {
+ ")</a> created a new application named " + applicationName, null );
}
- em.refreshIndex();
+
return new ApplicationInfo( applicationId, appInfo.getName() );
}
@@ -2376,7 +2375,7 @@ public class ManagementServiceImpl implements ManagementService {
if ( sendEmail ) {
startOrganizationActivationFlow( organization );
}
- em.refreshIndex();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index 5eed002..abd77ee 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -16,39 +16,56 @@
*/
package org.apache.usergrid.services.notifications;
-import java.util.*;
-import com.codahale.metrics.*;
-import com.codahale.metrics.Timer;
-import com.google.inject.Injector;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.mq.Message;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PathQuery;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.entities.Device;
import org.apache.usergrid.persistence.entities.Notification;
import org.apache.usergrid.persistence.entities.Notifier;
import org.apache.usergrid.persistence.entities.Receipt;
+import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-import org.apache.usergrid.services.*;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+import org.apache.usergrid.services.AbstractCollectionService;
+import org.apache.usergrid.services.ServiceAction;
+import org.apache.usergrid.services.ServiceContext;
+import org.apache.usergrid.services.ServiceInfo;
+import org.apache.usergrid.services.ServiceManagerFactory;
+import org.apache.usergrid.services.ServiceParameter;
+import org.apache.usergrid.services.ServicePayload;
+import org.apache.usergrid.services.ServiceRequest;
+import org.apache.usergrid.services.ServiceResults;
+import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.entities.Device;
-import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
-import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
-import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.google.inject.Injector;
import rx.Observable;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
+import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+
public class NotificationsService extends AbstractCollectionService {
@@ -90,9 +107,8 @@ public class NotificationsService extends AbstractCollectionService {
postMeter = metricsService.getMeter(NotificationsService.class, "requests");
postTimer = metricsService.getTimer(this.getClass(), "execution_rest");
JobScheduler jobScheduler = new JobScheduler(sm,em);
- String name = ApplicationQueueManagerImpl.getQueueNames(props);
- QueueScopeFactory queueScopeFactory = getApplicationContext().getBean( Injector.class ).getInstance(QueueScopeFactory.class);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), name);
+ String name = ApplicationQueueManagerImpl.getQueueNames( props );
+ QueueScope queueScope = new QueueScopeImpl( name );
queueManagerFactory = getApplicationContext().getBean( Injector.class ).getInstance(QueueManagerFactory.class);
QueueManager queueManager = TEST_QUEUE_MANAGER !=null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
notificationQueueManager = new ApplicationQueueManagerImpl(jobScheduler,em,queueManager,metricsService,props);
[21/50] incubator-usergrid git commit: Changes the batch future set.
Posted by to...@apache.org.
Changes the batch future set.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d81dfaa1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d81dfaa1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d81dfaa1
Branch: refs/heads/USERGRID-405
Commit: d81dfaa14f11c524e3f114b0b030bc447ee8100b
Parents: 77542c6
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:09:48 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:09:48 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d81dfaa1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 8481dab..16b3ff9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -197,9 +197,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* No-op, just disregard it
*/
if(tempContainer.isEmpty()){
- final BetterFuture<?> future = tempContainer.getFuture();
- future.done();
- return future;
+ tempContainer.done();
+ return tempContainer.getFuture();
}
return indexBatchBufferProducer.put(tempContainer);
[37/50] incubator-usergrid git commit: add replay strategy
Posted by to...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/117c0f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/117c0f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/117c0f3d
Branch: refs/heads/USERGRID-405
Commit: 117c0f3d8839cb1da2cae35b9a1171c1b404ac98
Parents: 4414838
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:10:55 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:10:55 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 94 +++++++++-----------
1 file changed, 42 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/117c0f3d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 5f96b92..173efa7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -101,9 +101,46 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final CollectionScope scope = ioevent.getEntityCollection();
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+ //allocate our max size, worst case
+ final List<Field> uniqueFields = new ArrayList<>( entity.getFields().size() );
+
+ //
+ // Construct all the functions for verifying we're unique
+ //
+ for ( final Field field : entity.getFields() ) {
+
+ // if it's unique, create a function to validate it and add it to the list of
+ // concurrent validations
+ if ( field.isUnique() ) {
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
+
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+
+ batch.mergeShallow( mb );
+ uniqueFields.add(field);
+ }
+ }
+
+ //short circuit nothing to do
+ if ( uniqueFields.size() == 0 ) {
+ return ;
+ }
+
+ //perform the write
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException ex ) {
+ throw new RuntimeException( "Unable to write to cassandra", ex );
+ }
+
// use simple thread pool to verify fields in parallel
- ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,keyspace,serializationFig,cassandraFig,scope,entity);
+ ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,cassandraFig,scope, uniqueFields,entity);
Map<String,Field> uniquenessViolations = cmd.execute();
+ cmd.getFailedExecutionException();
//We have violations, throw an exception
if ( !uniquenessViolations.isEmpty() ) {
throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
@@ -113,19 +150,17 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
private static class ConsistentReplayCommand extends HystrixCommand<Map<String,Field>>{
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final Keyspace keySpace;
- private final SerializationFig serializationFig;
private final CassandraConfig fig;
private final CollectionScope scope;
+ private final List<Field> uniqueFields;
private final Entity entity;
- public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy,Keyspace keySpace, SerializationFig serializationFig, CassandraConfig fig,CollectionScope scope, Entity entity){
+ public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy, CassandraConfig fig, CollectionScope scope, List<Field> uniqueFields, Entity entity){
super(REPLAY_GROUP);
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
- this.keySpace = keySpace;
- this.serializationFig = serializationFig;
this.fig = fig;
this.scope = scope;
+ this.uniqueFields = uniqueFields;
this.entity = entity;
}
@@ -142,23 +177,17 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
Collection<Field> entityFields = entity.getFields();
//allocate our max size, worst case
- final List<Field> uniqueFields = new ArrayList<Field>( entityFields.size() );
//now get the set of fields back
final UniqueValueSet uniqueValues;
- //todo add consistencylevel and read back if fail using
-
try {
-
- uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, entityFields );
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
}
-
final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
-
//loop through each field that was unique
for ( final Field field : uniqueFields ) {
@@ -170,51 +199,12 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
field.getName() ) );
}
-
final Id returnedEntityId = uniqueValue.getEntityId();
-
if ( !entity.getId().equals(returnedEntityId) ) {
uniquenessViolations.put( field.getName(), field );
}
}
- final MutationBatch batch = keySpace.prepareMutationBatch();
- //
- // Construct all the functions for verifying we're unique
- //
- for ( final Field field : entity.getFields() ) {
-
- // if it's unique, create a function to validate it and add it to the list of
- // concurrent validations
- if ( field.isUnique() ) {
-
-
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, entity.getId(), entity.getVersion() );
-
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueSerializationStrategy.write( scope, written, serializationFig.getTimeout() );
-
- batch.mergeShallow( mb );
-
-
- uniqueFields.add(field);
- }
- }
-
- //short circuit nothing to do
- if ( uniqueFields.size() == 0 ) {
- return uniquenessViolations ;
- }
-
-
- //perform the write
- try {
- batch.execute();
- }
- catch ( ConnectionException ex ) {
- throw new RuntimeException( "Unable to write to cassandra", ex );
- }
return uniquenessViolations;
}
[25/50] incubator-usergrid git commit: Fixes worker issue in cloud
formation
Posted by to...@apache.org.
Fixes worker issue in cloud formation
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c319d07d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c319d07d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c319d07d
Branch: refs/heads/USERGRID-405
Commit: c319d07d8c09d2e5988f58faad1dd42cff85cbe8
Parents: 9aec790
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 19:33:54 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 19:33:54 2015 -0600
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 8 +-
stack/awscluster/ugcluster-cf.json | 150 ++-----------------
2 files changed, 23 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 8af5884..fb98368 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -51,6 +51,9 @@ def esShards = numEsNodes*2;
def esReplicas = 1;
def tomcatThreads = System.getenv().get("TOMCAT_THREADS")
+
+def workerCount = System.getenv().get("INDEX_WORKER_COUNT")
+
//temporarily set to equal since we now have a sane tomcat thread calculation
def hystrixThreads = tomcatThreads
@@ -58,7 +61,6 @@ def hystrixThreads = tomcatThreads
def ec2Region = System.getenv().get("EC2_REGION")
def cassEc2Region = ec2Region.replace("-1", "")
-
NodeRegistry registry = new NodeRegistry();
def selectResult = registry.searchNode('cassandra')
@@ -195,6 +197,10 @@ usergrid.queue.region=${ec2Region}
usergrid.scheduler.enabled=true
usergrid.scheduler.job.workers=1
+
+#Set our ingest rate
+elasticsearch.worker_count=${workerCount}
+
"""
println usergridConfig
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 0c2a9d2..e9db671 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -46,6 +46,12 @@
],
"ConstraintDescription": "must be valid instance type."
},
+ "RestIndexWorkers":{
+ "Description": "The number of index workers to ingest ElasticSearch batch operations per tomcat",
+ "Type": "Number",
+ "Default": "8",
+ "MinValue": "3"
+ },
"TomcatThreadsPerCore": {
"Description": "Number of threads to configure tomcat for per core",
"Type": "Number",
@@ -289,7 +295,7 @@
}
}
},
- "GraphiteAutoScalingLaunchConfiguration":{
+ "GraphiteAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -464,7 +470,7 @@
}
}
},
- "OpsCenterUser": {
+ "OpsCenterUser": {
"Type": "AWS::IAM::User",
"Properties": {
"Path": "/",
@@ -485,7 +491,7 @@
]
}
},
- "OpsCenterKey": {
+ "OpsCenterKey": {
"Type": "AWS::IAM::AccessKey",
"Properties": {
"UserName": {
@@ -493,7 +499,7 @@
}
}
},
- "OpsCenterAutoScalingLaunchConfiguration":{
+ "OpsCenterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -619,7 +625,7 @@
}
},
- "OpsCenterAutoScalingGroup": {
+ "OpsCenterAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2014-07-24",
"Properties": {
@@ -767,7 +773,7 @@
}
}
},
- "CassAutoScalingLaunchConfiguration":{
+ "CassAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -955,7 +961,7 @@
}
}
},
- "ESMasterAutoScalingLaunchConfiguration":{
+ "ESMasterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -1137,131 +1143,6 @@
}
}
},
- "RestAutoScalingLaunchConfiguration":{
- "Type":"AWS::AutoScaling::LaunchConfiguration",
- "Properties":{
- "UserData":{
- "Fn::Base64":{
- "Fn::Join":[
- "",
- [
- "#!/bin/bash -ex\n",
- "# REST SERVER STARTUP \n",
- "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
- "\n",
- "mkdir -p /usr/share/usergrid\n",
- "\n",
- "# create script that sets our environment variables\n",
- "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
- "alias sudo='sudo -E'\n",
- "\n",
- "export TYPE=rest\n",
- "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n",
- "export YOURKIT=", { "Ref":"InstallYourkit" }, "\n",
- "export DNS_NAME=", { "Ref":"DnsSubDomain" }, "\n",
- "export DNS_DOMAIN=", { "Ref":"DnsDomain" }, "\n",
- "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
- "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
- "export ELB_NAME=", { "Ref":"RestElasticLoadBalancer" }, "\n",
- "\n",
- "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
- "export EC2_REGION=", { "Ref":"AWS::Region" }, "\n",
- "export EC2_URL=https://ec2.amazonaws.com/\n", "\n",
- "export REST_SECURITY_GROUP_NAME=", { "Ref":"RestSecurityGroup" }, "\n",
- "export DB_SECURITY_GROUP_NAME=", { "Ref":"CassSecurityGroup" }, "\n",
- "\n",
- "export CASSANDRA_CLUSTER_NAME=", { "Ref":"CassClusterName" }, "\n",
- "export CASSANDRA_KEYSPACE_NAME=usergrid", "\n",
- "export CASSANDRA_NUM_SERVERS=", { "Ref":"CassNumServers" }, "\n",
- "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
- "export TOMCAT_NUM_SERVERS=", { "Ref":"RestMinServers" }, "\n",
- "\n",
- "export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor" }, "\n",
- "\n",
- "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency" }, "\n",
- "\n",
- "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
- "\n",
-
- "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
- "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
- "\n",
- "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
- "\n",
- "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
- "\n",
- "export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
- "export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
- "\n",
- "EOF\n",
- "\n",
- "# put AWS creds in environment\n",
- "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
- "export AWS_ACCESS_KEY=", { "Ref":"RestKey" }, "\n",
- "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
- "EOF\n",
- "\n",
- "# setup s3cmd (will be installed by init script) \n",
- "cat >/etc/s3cfg <<EOF\n",
- "access_key=", { "Ref":"RestKey" }, "\n",
- "secret_key=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
- "EOF\n",
- "chmod 644 /etc/s3cfg\n",
- "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
- "ln -s /etc/s3cfg ~root/.s3cfg\n",
- "\n",
- "# download usergrid and init script bundle from S3\n",
- "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
- "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
- "apt-get update\n",
- "apt-get -y install s3cmd\n",
- "cd /usr/share/usergrid\n",
- "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/ROOT.war\n",
- "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "mv ROOT.war webapps/ROOT.war\n",
- "chmod 755 ./init_instance/*.sh\n",
- "cd ./init_instance\n",
- "# Init as a REST intance \n",
- "sh ./init_rest_server.sh\n"
- ]
- ]
- }
- },
- "KeyName":{
- "Ref":"KeyPair"
- },
- "ImageId":{
- "Fn::FindInMap":[
- "AWSRegionArch2AMI",
- {
- "Ref":"AWS::Region"
- },
- {
- "Fn::FindInMap":[
- "AWSInstanceType2Arch",
- {
- "Ref":"RestInstanceType"
- },
- "Arch"
- ]
- }
- ]
- },
- "InstanceType":{
- "Ref":"RestInstanceType"
- },
- "IamInstanceProfile":{
- "Ref":"RootInstanceProfile"
- },
- "SecurityGroups":[
- {
- "Ref":"RestSecurityGroup"
- }
- ]
- }
- },
"ESAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
@@ -1378,7 +1259,7 @@
]
}
},
- "ESAutoScalingGroup": {
+ "ESAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
"Properties": {
@@ -1442,7 +1323,7 @@
}
}
},
- "RestAutoScalingLaunchConfiguration":{
+ "RestAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -1487,6 +1368,7 @@
"\n",
"export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
"\n",
+ "export INDEX_WORKER_COUNT=", { "Ref":"RestIndexWorkers" }, "\n",
"export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
"export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
[31/50] incubator-usergrid git commit: change consistency level
Posted by to...@apache.org.
change consistency level
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9e6e36dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9e6e36dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9e6e36dd
Branch: refs/heads/USERGRID-405
Commit: 9e6e36ddb5bd1d38622b100b4916cac6dcbc1d77
Parents: e8dc17d
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 12 17:29:48 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 12 17:29:48 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 3 +-
.../UniqueValueSerializationStrategy.java | 20 ++++++++++----
.../UniqueValueSerializationStrategyImpl.java | 19 ++++++++-----
.../core/astyanax/CassandraConfig.java | 6 ++++
.../core/astyanax/CassandraConfigImpl.java | 8 ++++--
.../persistence/core/astyanax/CassandraFig.java | 9 ++++--
.../core/astyanax/ColumnNameIteratorTest.java | 5 ++++
.../MultiKeyColumnNameIteratorTest.java | 5 ++++
.../astyanax/MultiRowColumnIteratorTest.java | 5 ++++
.../index/impl/EsEntityIndexImpl.java | 29 +++++++++++---------
.../persistence/queue/QueueManagerTest.java | 2 +-
11 files changed, 80 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 162bcac..1c30e75 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,7 +111,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
//
for ( final Field field : entityFields ) {
- // if it's unique, create a function to validate it and add it to the list of
+ // if it's unique, create a function to validate it and add it to the list of
// concurrent validations
if ( field.isUnique() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 030d9d1..8d314d8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.ConsistencyLevel;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.field.Field;
@@ -34,7 +35,7 @@ public interface UniqueValueSerializationStrategy extends Migration {
/**
* Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds.
- *
+ *
* @param uniqueValue Object to be written
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
@@ -42,16 +43,16 @@ public interface UniqueValueSerializationStrategy extends Migration {
/**
* Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds.
- *
+ *
* @param uniqueValue Object to be written
- * @param timeToLive How long object should live in seconds
+ * @param timeToLive How long object should live in seconds
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
public MutationBatch write( CollectionScope scope, UniqueValue uniqueValue, Integer timeToLive );
/**
* Load UniqueValue that matches field from collection or null if that value does not exist.
- *
+ *
* @param colScope Collection scope in which to look for field name/value
* @param fields Field name/value to search for
* @return UniqueValueSet containing fields from the collection that exist in cassandra
@@ -60,8 +61,17 @@ public interface UniqueValueSerializationStrategy extends Migration {
public UniqueValueSet load( CollectionScope colScope, Collection<Field> fields ) throws ConnectionException;
/**
+ * Load UniqueValue that matches field from collection or null if that value does not exist.
+ *
+ * @param colScope Collection scope in which to look for field name/value
+ * @param fields Field name/value to search for
+ * @return UniqueValueSet containing fields from the collection that exist in cassandra
+ * @throws ConnectionException on error connecting to Cassandra
+ */
+ public UniqueValueSet load( CollectionScope colScope, ConsistencyLevel consistencyLevel, Collection<Field> fields ) throws ConnectionException;
+ /**
* Delete the specified Unique Value from Cassandra.
- *
+ *
* @param uniqueValue Object to be deleted.
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index be95b08..6483408 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -25,6 +25,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import org.apache.usergrid.persistence.core.astyanax.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,10 +36,6 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
-import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
-import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -72,6 +70,7 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
ENTITY_VERSION_SER );
protected final Keyspace keyspace;
+ private final CassandraFig cassandraFig;
/**
@@ -80,7 +79,8 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
* @param keyspace Keyspace in which to store Unique Values.
*/
@Inject
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace ) {
+ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig) {
+ this.cassandraFig = cassandraFig;
this.keyspace = keyspace;
}
@@ -171,8 +171,13 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
}
+
+ @Override
+ public UniqueValueSet load(final CollectionScope colScope, final Collection<Field> fields ) throws ConnectionException{
+ return load(colScope,ConsistencyLevel.valueOf(cassandraFig.getReadCL()), fields);
+ }
@Override
- public UniqueValueSet load(final CollectionScope colScope, final Collection<Field> fields )
+ public UniqueValueSet load(final CollectionScope colScope, final ConsistencyLevel consistencyLevel, final Collection<Field> fields )
throws ConnectionException {
Preconditions.checkNotNull( fields, "fields are required" );
@@ -198,7 +203,7 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl( fields.size() );
Iterator<Row<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion>> results =
- keyspace.prepareQuery( CF_UNIQUE_VALUES ).getKeySlice( keys )
+ keyspace.prepareQuery( CF_UNIQUE_VALUES ).setConsistencyLevel(consistencyLevel).getKeySlice( keys )
.withColumnRange( new RangeBuilder().setLimit( 1 ).build() ).execute().getResult().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
index ad4463a..817aee2 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
@@ -37,6 +37,12 @@ public interface CassandraConfig {
public ConsistencyLevel getReadCL();
/**
+ * Get the currently configured ReadCL that is more consitent than getReadCL
+ * @return
+ */
+ public ConsistencyLevel getConsistentReadCL();
+
+ /**
* Get the currently configured write CL
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
index ce80ea2..17b91c6 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
@@ -39,6 +39,7 @@ public class CassandraConfigImpl implements CassandraConfig {
private ConsistencyLevel readCl;
private ConsistencyLevel writeCl;
private int[] shardSettings;
+ private ConsistencyLevel consistentCl;
@Inject
@@ -50,7 +51,7 @@ public class CassandraConfigImpl implements CassandraConfig {
this.shardSettings = parseShardSettings( cassandraFig.getShardValues() );
-
+ this.consistentCl = ConsistencyLevel.valueOf(cassandraFig.getConsistentReadCL());
//add the listeners to update the values
cassandraFig.addPropertyChangeListener( new PropertyChangeListener() {
@@ -78,7 +79,10 @@ public class CassandraConfigImpl implements CassandraConfig {
return readCl;
}
-
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return consistentCl;
+ }
@Override
public ConsistencyLevel getWriteCL() {
return writeCl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index 1208a1a..66a716c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -33,6 +33,8 @@ import org.safehaus.guicyfig.Key;
public interface CassandraFig extends GuicyFig {
+ public static final String READ_CONSISTENT_CL = "usergrid.consistent.read.cl";
+
public static final String READ_CL = "usergrid.read.cl";
public static final String WRITE_CL = "usergrid.write.cl";
@@ -77,11 +79,14 @@ public interface CassandraFig extends GuicyFig {
@Default( "false" )
boolean isEmbedded();
-
- @Default("CL_QUORUM")
+ @Default("CL_ONE")
@Key(READ_CL)
String getReadCL();
+ @Default("CL_LOCAL_ONE")
+ @Key(READ_CONSISTENT_CL)
+ String getConsistentReadCL();
+
@Default("CL_QUORUM")
@Key(WRITE_CL)
String getWriteCL();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
index 547c504..0b0cbab 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -79,6 +79,11 @@ public class ColumnNameIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
index a96db39..964c04a 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -83,6 +83,11 @@ public class MultiKeyColumnNameIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
index 4530d4b..11d34a4 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -86,6 +86,11 @@ public class MultiRowColumnIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fa50734..9343d39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -113,6 +113,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer searchTimer;
private final Timer allVersionsTimerFuture;
private final Timer deletePreviousTimerFuture;
+ private final Meter errorMeter;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -174,31 +175,33 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.aliasCache = indexCache;
this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.timer" );
this.removeAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "remove.alias.timer" );
this.addReadAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.read.alias.timer" );
this.addWriteAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.write.alias.timer" );
this.mappingTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" );
+ .getTimer( EsEntityIndexImpl.class, "create.mapping.timer" );
this.refreshTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" );
+ .getTimer( EsEntityIndexImpl.class, "refresh.timer" );
this.searchTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer.future" );
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer.future" );
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
+
+ this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index 935fd16..e20f87c 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -63,7 +63,7 @@ public class QueueManagerTest {
@Before
public void mockApp() {
this.scope = new QueueScopeImpl( new SimpleId( "application" ), "testQueue" );
- qm = qmf.getQueueManager(scope);
+// qm = qmf.getQueueManager(scope);
queueScopeFactory = new QueueScopeFactoryImpl(queueFig);
}
[42/50] incubator-usergrid git commit: Applied complains and
refactored names to make the code clearer and checking for more null
pointers. Removed redundant test.
Posted by to...@apache.org.
Applied complains and refactored names to make the code clearer and checking for more null pointers.
Removed redundant test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b0a07baa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b0a07baa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b0a07baa
Branch: refs/heads/USERGRID-405
Commit: b0a07baa354f790d820d1a32b435502ef5b062a9
Parents: 24a48a7
Author: GERey <gr...@apigee.com>
Authored: Fri Mar 13 15:23:15 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Mar 13 15:23:15 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 10 ++-
.../usergrid/persistence/EntityManager.java | 5 +-
.../cassandra/EntityManagerImpl.java | 2 +-
.../usergrid/persistence/EntityManagerIT.java | 23 -------
.../collection/EntityCollectionManager.java | 10 +--
.../cache/CachedEntityCollectionManager.java | 17 +-----
.../impl/EntityCollectionManagerImpl.java | 15 +++--
.../serialization/impl/FieldSetImpl.java | 64 --------------------
.../serialization/impl/MutableFieldSet.java | 63 +++++++++++++++++++
.../collection/EntityCollectionManagerIT.java | 7 +--
.../services/AbstractCollectionService.java | 10 +--
.../services/AbstractConnectionsService.java | 6 +-
12 files changed, 100 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a108f25..e5223f0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -809,7 +809,7 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Entity getAllEntityFromFields( String collectionType, String aliasType ){
+ public Entity getUniqueEntityFromAlias( String collectionType, String aliasType ){
String collName = Schema.defaultCollectionName( collectionType );
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
@@ -824,8 +824,13 @@ public class CpEntityManager implements EntityManager {
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
- Observable<FieldSet> fieldSetObservable = ecm.getAllEntities( Arrays.<Field>asList( uniqueLookupRepairField) );
+ Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields(
+ Arrays.<Field>asList( uniqueLookupRepairField ) );
+ if(fieldSetObservable == null){
+ logger.debug( "Couldn't return the observable based on unique entities." );
+ return null;
+ }
FieldSet fieldSet = fieldSetObservable.toBlocking().last();
repairedEntityGet.stop();
@@ -2284,7 +2289,6 @@ public class CpEntityManager implements EntityManager {
getApplicationScope().getApplication(), collectionName);
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
-//TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 5657d4f..65fac8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -713,6 +713,5 @@ public interface EntityManager {
*/
public Health getIndexHealth();
- public Entity getAllEntityFromFields( String aliasType, String aliasValue );
-
- }
+ public Entity getUniqueEntityFromAlias( String aliasType, String aliasValue );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 3993285..e505b8e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2949,7 +2949,7 @@ public class EntityManagerImpl implements EntityManager {
@Override
- public Entity getAllEntityFromFields( final String aliasType, final String aliasValue ) {
+ public Entity getUniqueEntityFromAlias( final String aliasType, final String aliasValue ) {
throw new UnsupportedOperationException( "Not supported." );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index cd08607..4896fc1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -579,27 +579,4 @@ public class EntityManagerIT extends AbstractCoreIT {
assertNotNull( em.get( user.getUuid() ));
}
-
- @Test
- public void testInvalidNameRepair() throws Exception {
- LOG.info("EntityManagerIT.testInvalidNameReapir");
-
- EntityManager em = app.getEntityManager();
-
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "name", "XR-51B" );
- properties.put( "fuel", "Nutrinox" );
-
- Entity user = em.create( "robot", properties );
- assertNotNull( user );
-
- em.refreshIndex();
-
- assertNotNull( em.get( user.getUuid() ));
-
- em.delete( user );
-
- assertNull( em.get( user.getUuid() ));
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 6bbe912..35fc5d4 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -54,9 +54,9 @@ public interface EntityCollectionManager {
* Return the latest versions of the specified entityIds
*/
public Observable<VersionSet> getLatestVersion( Collection<Id> entityId );
-
-
- public Observable<FieldSet> getAllEntities(Collection<Field> fields);
+
+
+ public Observable<FieldSet> getEntitiesFromFields( Collection<Field> fields );
/**
* Gets the Id for a field
@@ -77,12 +77,12 @@ public interface EntityCollectionManager {
/**
* Takes the change and reloads an entity with all changes applied in this entity applied.
- * The resulting entity from calling load will be the previous version of this entity plus
+ * The resulting entity from calling load will be the previous version of this entity plus
* the entity in this object applied to it.
*/
public Observable<Entity> update ( Entity entity );
- /**
+ /**
* Returns health of entity data store.
*/
public Health getHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
index d33f762..7d229e6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
@@ -20,19 +20,10 @@
package org.apache.usergrid.persistence.collection.cache;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import org.apache.usergrid.persistence.collection.*;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.collection.serialization.impl.FieldSetImpl;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -40,15 +31,11 @@ import org.apache.usergrid.persistence.model.field.Field;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import rx.Observable;
import rx.functions.Action1;
-import rx.functions.Func1;
@Singleton
@@ -83,8 +70,8 @@ public class CachedEntityCollectionManager implements EntityCollectionManager {
}
@Override
- public Observable<FieldSet> getAllEntities(final Collection<Field> fields) {
- return targetEntityCollectionManager.getAllEntities( fields );
+ public Observable<FieldSet> getEntitiesFromFields( final Collection<Field> fields ) {
+ return targetEntityCollectionManager.getEntitiesFromFields( fields );
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 7db34aa..3c89280 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,8 +23,8 @@ import java.util.*;
import com.netflix.astyanax.MutationBatch;
import org.apache.usergrid.persistence.collection.*;
-import org.apache.usergrid.persistence.collection.serialization.impl.FieldSetImpl;
-import org.apache.usergrid.persistence.core.rx.ObservableIterator;
+import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +41,6 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
@@ -294,7 +293,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
* @return
*/
@Override
- public Observable<FieldSet> getAllEntities(final Collection<Field> fields) {
+ public Observable<FieldSet> getEntitiesFromFields( final Collection<Field> fields ) {
return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>() {
@Override
public FieldSet call( Collection<Field> fields ) {
@@ -307,7 +306,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
//Short circut if we don't have any uniqueValues from the given fields.
if(!set.iterator().hasNext()){
- return new FieldSetImpl( 0 );
+ return new MutableFieldSet( 0 );
}
@@ -319,6 +318,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
UniqueValue value = set.getValue(expectedField.getName());
+ if(value ==null){
+ logger.debug( "Field does not correspond to a unique value" );
+ }
+
entityIds.add(value.getEntityId());
uniqueValues.add(value);
}
@@ -329,7 +332,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
//now loop through and ensure the entities are there.
final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
- final FieldSetImpl response = new FieldSetImpl(fields.size());
+ final MutableFieldSet response = new MutableFieldSet(fields.size());
for(final UniqueValue expectedUnique: uniqueValues) {
final MvccEntity entity = entitySet.getEntity(expectedUnique.getEntityId());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
deleted file mode 100644
index 241d5c2..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/FieldSetImpl.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.apache.usergrid.persistence.collection.serialization.impl;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.model.field.Field;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-public class FieldSetImpl implements FieldSet {
-
-
- private final Map<Field<?>, MvccEntity> entities;
-
-
- public FieldSetImpl(
- final int expectedSize) {
- this.entities = new HashMap<>( expectedSize );
- }
-
-
- public void addEntity(final Field<?> field, final MvccEntity entity ) {
- entities.put( field, entity );
- }
-
-
- @Override
- public MvccEntity getEntity( final Field<?> field) {
- return entities.get( field );
- }
-
-
-
-
- @Override
- public int size() {
- return entities.size();
- }
-
-
- @Override
- public boolean isEmpty() {
- return entities.size() == 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
new file mode 100644
index 0000000..ae921a7
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
@@ -0,0 +1,63 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.collection.FieldSet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.model.field.Field;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class MutableFieldSet implements FieldSet {
+
+
+ private final Map<Field<?>, MvccEntity> entities;
+
+
+ public MutableFieldSet( final int expectedSize ) {
+ this.entities = new HashMap<>( expectedSize );
+ }
+
+
+ public void addEntity(final Field<?> field, final MvccEntity entity ) {
+ entities.put( field, entity );
+ }
+
+
+ @Override
+ public MvccEntity getEntity( final Field<?> field) {
+ return entities.get( field );
+ }
+
+
+
+
+ @Override
+ public int size() {
+ return entities.size();
+ }
+
+
+ @Override
+ public boolean isEmpty() {
+ return entities.size() == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 46e844d..a52486e 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence.collection;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -780,12 +779,12 @@ public class EntityCollectionManagerIT {
Entity createReturned = observable.toBlocking().lastOrDefault( null );
-
+ assertNotNull( "Need entity to be created before proceeding", createReturned );
assertNotNull( "Id was assigned", createReturned.getId() );
assertNotNull( "Version was assigned", createReturned.getVersion() );
FieldSet
- fieldResults = manager.getAllEntities( Arrays.<Field>asList( expectedInteger) ).toBlocking().last();
+ fieldResults = manager.getEntitiesFromFields( Arrays.<Field>asList( expectedInteger ) ).toBlocking().last();
assertEquals(1,fieldResults.size());
@@ -800,7 +799,7 @@ public class EntityCollectionManagerIT {
//try to load via the unique field, should have triggered repair
final FieldSet
- results = manager.getAllEntities( Arrays.<Field>asList( expectedInteger) ).toBlocking().last();
+ results = manager.getEntitiesFromFields( Arrays.<Field>asList( expectedInteger ) ).toBlocking().last();
//verify no entity returned
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index e404321..dcd7557 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -72,7 +72,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name );
if ( entity != null ) {
entity = importEntity( request, entity );
}
@@ -144,7 +144,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name );
if ( entity == null ) {
logger.info( "miss on entityType: {} with name: {}", getEntityType(), name );
@@ -287,7 +287,7 @@ public class AbstractCollectionService extends AbstractService {
}
// EntityRef ref = em.getAlias( getEntityType(), name );
- Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name );
if ( entity == null ) {
// null entity ref means we tried to put a non-existing entity
// before we create a new entity for it, we should check for permission
@@ -431,7 +431,7 @@ public class AbstractCollectionService extends AbstractService {
return super.postItemByName( context, name );
}
- Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
@@ -485,7 +485,7 @@ public class AbstractCollectionService extends AbstractService {
return getItemByName( context, name );
}
- Entity entity = em.getAllEntityFromFields(getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0a07baa/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index 8a95e5c..1e4c4b8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -265,7 +265,7 @@ public class AbstractConnectionsService extends AbstractService {
//TODO T.N. USERGRID-1919 actually validate this is connected
- Entity entity = em.getAllEntityFromFields( query.getEntityType(),name );
+ Entity entity = em.getUniqueEntityFromAlias( query.getEntityType(), name );
if ( entity == null ) {
return null;
}
@@ -361,7 +361,7 @@ public class AbstractConnectionsService extends AbstractService {
if ( query.containsSingleNameOrEmailIdentifier() ) {
String name = query.getSingleNameOrEmailIdentifier();
- entity = em.getAllEntityFromFields( query.getEntityType(), name );
+ entity = em.getUniqueEntityFromAlias( query.getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
@@ -472,7 +472,7 @@ public class AbstractConnectionsService extends AbstractService {
nameProperty = "name";
}
- Entity entity = em.getAllEntityFromFields( query.getEntityType(), name );
+ Entity entity = em.getUniqueEntityFromAlias( query.getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
[45/50] incubator-usergrid git commit: Fixes bug in writeunique
verify failover command
Posted by to...@apache.org.
Fixes bug in writeunique verify failover command
Fixes NoAWSCredsRule to use Assumptions to mark as ignored
Adds AWS creds rule to NotificationsServiceTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/03341191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/03341191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/03341191
Branch: refs/heads/USERGRID-405
Commit: 03341191a2109b8d297387c37b88df7af7b50218
Parents: f20dfd3
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 15:34:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 15:34:45 2015 -0600
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 3 ++-
.../impl/EntityVersionCleanupTask.java | 7 +++--
.../mvcc/stage/write/WriteUniqueVerify.java | 24 +++++++++--------
.../collection/util/EntityUtils.java | 28 +++++++++++++++++---
.../index/impl/EsIndexBufferConsumerImpl.java | 14 ++++++++--
.../persistence/queue/NoAWSCredsRule.java | 17 ++++++++++++
stack/services/pom.xml | 13 ++++++++-
.../notifications/NotifiersServiceIT.java | 6 +++++
8 files changed, 90 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ac04dcc..4bde50e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -392,9 +392,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
+ queryCollectionEm("dogs", "select *");
Thread.sleep(100);
crs = queryCollectionCp("dogs", "dog", "select *");
- } while ( crs.size() == numEntities && count++ < 15 );
+ } while ( crs.size() != numEntities && count++ < 15 );
Assert.assertEquals("Expect candidates without earlier stale entities", crs.size(), numEntities);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 03a5ff6..65506ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
@@ -158,10 +159,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
final Entity entity = mvccEntity.getEntity().get();
//remove all unique fields from the index
- for ( final Field field : entity.getFields() ) {
- if ( !field.isUnique() ) {
- continue;
- }
+ for ( final Field field : EntityUtils.getUniqueFields(entity )) {
+
final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
final MutationBatch deleteMutation =
uniqueValueSerializationStrategy.delete( scope, unique );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 173efa7..5bdf3b9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -31,6 +31,8 @@ import com.netflix.hystrix.HystrixCommand;
import com.netflix.hystrix.HystrixCommandGroupKey;
import com.netflix.hystrix.HystrixThreadPoolProperties;
import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.slf4j.Logger;
@@ -108,20 +110,21 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
//
// Construct all the functions for verifying we're unique
//
- for ( final Field field : entity.getFields() ) {
+
+
+ for ( final Field field : EntityUtils.getUniqueFields(entity)) {
// if it's unique, create a function to validate it and add it to the list of
// concurrent validations
- if ( field.isUnique() ) {
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- batch.mergeShallow( mb );
- uniqueFields.add(field);
- }
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+
+ batch.mergeShallow( mb );
+ uniqueFields.add(field);
}
//short circuit nothing to do
@@ -175,12 +178,11 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
- Collection<Field> entityFields = entity.getFields();
//allocate our max size, worst case
//now get the set of fields back
final UniqueValueSet uniqueValues;
try {
- uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, entityFields );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index 6233075..cf964a3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@ -1,8 +1,11 @@
package org.apache.usergrid.persistence.collection.util;
-import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.commons.lang3.reflect.FieldUtils;
@@ -16,9 +19,9 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class EntityUtils {
- private static final Field VERSION = FieldUtils.getField( Entity.class, "version", true );
+ private static final java.lang.reflect.Field VERSION = FieldUtils.getField( Entity.class, "version", true );
- private static final Field ID = FieldUtils.getField( Entity.class, "id", true );
+ private static final java.lang.reflect.Field ID = FieldUtils.getField( Entity.class, "id", true );
/**
@@ -46,4 +49,23 @@ public class EntityUtils {
throw new RuntimeException( "Unable to set the field " + ID + " into the entity", e );
}
}
+
+
+ /**
+ * Get the unique fields for an entity
+ * @param entity
+ * @return
+ */
+ public static List<Field> getUniqueFields( final Entity entity ){
+
+ final List<Field> uniqueFields = new ArrayList<>(entity.getFields().size());
+
+ for(final Field field: entity.getFields()){
+ if(field.isUnique()){
+ uniqueFields.add( field);
+ }
+ }
+
+ return uniqueFields;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index d55073a..b31cf39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -322,8 +322,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
for (BulkItemResponse response : responses) {
if (response.isFailed()) {
- throw new RuntimeException("Unable to index documents. Errors are :"
- + response.getFailure().getMessage());
+
+ final BulkItemResponse.Failure failure = response.getFailure();
+
+ final String message;
+
+ if(failure != null) {
+ message = "Unable to index documents. Errors are :" + response.getFailure().getMessage();
+ }else{
+ message = "Unable to index documents. Response is :" + response.getResponse();
+ }
+
+ throw new RuntimeException(message);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
index 45218c2..ba0dc1f 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.persistence.queue;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
@@ -49,6 +51,9 @@ public class NoAWSCredsRule implements TestRule {
throw t;
}
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
}
}
@@ -69,6 +74,18 @@ public class NoAWSCredsRule implements TestRule {
}
}
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
final Throwable cause = t.getCause();
if ( cause == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index f5e8dd5..68f1a4a 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -133,6 +133,7 @@
<scope>runtime</scope>
</dependency>
+
<dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-core</artifactId>
@@ -290,7 +291,15 @@
<classifier>tests</classifier>
</dependency>
- <dependency>
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-config</artifactId>
<version>${project.version}</version>
@@ -311,6 +320,8 @@
<scope>test</scope>
</dependency>
+
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index 3c1c645..e6970b5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -17,12 +17,15 @@
package org.apache.usergrid.services.notifications;
import org.apache.commons.io.IOUtils;
+
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
import org.apache.usergrid.persistence.entities.Notifier;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.apache.usergrid.services.notifications.ConnectionException;
import org.apache.usergrid.services.notifications.NotificationsService;
@@ -48,6 +51,9 @@ public class NotifiersServiceIT extends AbstractServiceIT {
private QueueListener listener;
+ @Rule
+ public NoAWSCredsRule noCredsRule = new NoAWSCredsRule();
+
@Before
public void before() throws Exception {
[09/50] incubator-usergrid git commit: remove yammer
Posted by to...@apache.org.
remove yammer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/62635942
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/62635942
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/62635942
Branch: refs/heads/USERGRID-405
Commit: 626359421e9cfedf0fd5baba767051d37b7a24ab
Parents: 89f558a
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 11:35:12 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 11:35:12 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 10 ----
.../batch/service/JobSchedulerService.java | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 4 --
.../corepersistence/CpRelationManager.java | 14 -----
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 23 +-------
.../cassandra/RelationManagerImpl.java | 58 +++-----------------
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
stack/pom.xml | 43 ---------------
stack/rest/pom.xml | 4 --
.../org/apache/usergrid/rest/RootResource.java | 6 --
stack/test-utils/pom.xml | 5 --
15 files changed, 10 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index d30bedb..3a40366 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -279,16 +279,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
index 5d57ab7..3674383 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
@@ -46,8 +46,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import com.yammer.metrics.annotation.ExceptionMetered;
-import com.yammer.metrics.annotation.Timed;
/**
@@ -81,8 +79,6 @@ public class JobSchedulerService extends AbstractScheduledService {
public JobSchedulerService() { }
- @Timed( name = "BulkJobScheduledService_runOneIteration", group = "scheduler", durationUnit = TimeUnit.MILLISECONDS,
- rateUnit = TimeUnit.MINUTES )
@Override
protected void runOneIteration() throws Exception {
@@ -143,7 +139,6 @@ public class JobSchedulerService extends AbstractScheduledService {
/**
* Use the provided BulkJobFactory to build and submit BulkJob items as ListenableFuture objects
*/
- @ExceptionMetered( name = "BulkJobScheduledService_submitWork_exceptions", group = "scheduler" )
private void submitWork( final JobDescriptor jobDescriptor ) {
final Job job;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 26eff4c..d808ae5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -108,7 +108,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.netflix.hystrix.exception.HystrixRuntimeException;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -388,7 +387,6 @@ public class CpEntityManager implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_create" )
@TraceParticipant
public <A extends Entity> A create( String entityType, Class<A> entityClass,
Map<String, Object> properties, UUID importId ) throws Exception {
@@ -1261,7 +1259,6 @@ public class CpEntityManager implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValues" )
public Map<String, Object> getDictionaryElementValues( EntityRef entity, String dictionaryName,
String... elementNames ) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 12504ed..fe4d828 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -21,7 +21,6 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.*;
@@ -391,20 +390,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications() throws Exception {
return getApplications( false );
}
@Override
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getDeletedApplications() throws Exception {
return getApplications( true );
}
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications(boolean deleted) throws Exception {
Map<String, UUID> appMap = new HashMap<String, UUID>();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index d018324..ee81a67 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -108,7 +108,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -490,7 +489,6 @@ public class CpRelationManager implements RelationManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "RelationManager_isOwner" )
@Override
public boolean isCollectionMember( String collName, EntityRef entity ) throws Exception {
@@ -742,7 +740,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createItemInCollection" )
public Entity createItemInCollection(
String collName, String itemType, Map<String, Object> properties) throws Exception {
@@ -1012,7 +1009,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connection_ref" )
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
return createConnection( connection.getConnectionType(), connection.getConnectedEntity() );
@@ -1020,7 +1016,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connectionType" )
public ConnectionRef createConnection( String connectionType, EntityRef connectedEntityRef ) throws Exception {
headEntity = em.validate( headEntity );
@@ -1091,7 +1086,6 @@ public class CpRelationManager implements RelationManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "CpRelationManager_batchUpdateEntityConnection" )
public Mutator<ByteBuffer> batchUpdateEntityConnection(
Mutator<ByteBuffer> batch,
boolean disconnect,
@@ -1200,7 +1194,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_paired_connection_type" )
public ConnectionRef createConnection(
String pairedConnectionType,
EntityRef pairedEntity,
@@ -1212,7 +1205,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connected_entity_ref" )
public ConnectionRef createConnection( ConnectedEntityRef... connections ) throws Exception {
throw new UnsupportedOperationException( "Paired connections not supported" );
@@ -1680,7 +1672,6 @@ public class CpRelationManager implements RelationManager {
* @return The indexUpdate with batch mutations
* @throws Exception the exception
*/
- @Metered( group = "core", name = "RelationManager_batchUpdateCollectionIndex" )
public IndexUpdate batchUpdateCollectionIndex(
IndexUpdate indexUpdate, EntityRef owner, String collectionName )
throws Exception {
@@ -1779,7 +1770,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchStartIndexUpdate")
public IndexUpdate batchStartIndexUpdate(
Mutator<ByteBuffer> batch, Entity entity, String entryName,
Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
@@ -1939,7 +1929,6 @@ public class CpRelationManager implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsDictionaryIndexes")
public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes(
IndexUpdate indexUpdate ) throws Exception {
@@ -2006,7 +1995,6 @@ public class CpRelationManager implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateConnectionIndex")
public IndexUpdate batchUpdateConnectionIndex(
IndexUpdate indexUpdate, ConnectionRefImpl connection ) throws Exception {
@@ -2135,7 +2123,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered( group = "core", name = "RelationManager_batchDeleteConnectionIndexEntries" )
public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries(
IndexUpdate indexUpdate,
IndexUpdate.IndexEntry entry,
@@ -2190,7 +2177,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered( group = "core", name = "RelationManager_batchAddConnectionIndexEntries" )
public Mutator<ByteBuffer> batchAddConnectionIndexEntries( IndexUpdate indexUpdate, IndexUpdate.IndexEntry entry,
ConnectionRefImpl conn, UUID[] index_keys ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 880fe89..6db05f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
-import com.yammer.metrics.annotation.Metered;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.springframework.context.ApplicationContext;
@@ -97,7 +96,6 @@ public interface EntityManagerFactory {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public abstract Map<String, UUID> getApplications() throws Exception;
public Map<String, UUID> getDeletedApplications() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index af1eabc..a8c62d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -40,7 +40,6 @@ import org.apache.commons.lang.StringUtils;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -274,7 +273,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
@Override
- @Metered(group = "core", name = "EntityManagerFactory_lookupApplication_byName")
public UUID lookupApplication( String name ) throws Exception {
name = name.toLowerCase();
HColumn<String, ByteBuffer> column =
@@ -295,7 +293,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Application getApplication( String name ) throws Exception {
name = name.toLowerCase();
HColumn<String, ByteBuffer> column =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index be43920..7dcbca7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -90,7 +90,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -191,6 +190,7 @@ public class EntityManagerImpl implements EntityManager {
public static final String APPLICATION_COLLECTION = "application.collection.";
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
+
@Resource
private EntityManagerFactoryImpl emf;
@Resource
@@ -505,7 +505,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @return True if this entity can safely "own" this property name and value unique combination
*/
- @Metered( group = "core", name = "EntityManager_isPropertyValueUniqueForEntity" )
public boolean isPropertyValueUniqueForEntity( UUID ownerEntityId, String entityType, String propertyName,
Object propertyValue ) throws Exception {
@@ -536,7 +535,6 @@ public class EntityManagerImpl implements EntityManager {
/**
* Return all UUIDs that have this unique value
*
- * @param ownerEntityId The entity id that owns this entity collection
* @param collectionName The entity collection name
* @param propertyName The name of the unique property
* @param propertyValue The value of the unique property
@@ -623,7 +621,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAlias_single" )
public EntityRef getAlias( EntityRef ownerRef, String collectionType, String aliasValue )
throws Exception {
@@ -658,7 +655,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAlias_multi" )
public Map<String, EntityRef> getAlias(
EntityRef ownerRef, String collectionName, List<String> aliases ) throws Exception {
@@ -735,7 +731,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_create" )
@TraceParticipant
public <A extends Entity> A create(
String entityType, Class<A> entityClass, Map<String, Object> properties,
@@ -754,7 +749,6 @@ public class EntityManagerImpl implements EntityManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "EntityManager_batchCreate" )
public <A extends Entity> A batchCreate(
Mutator<ByteBuffer> m, String entityType, Class<A> entityClass,
Map<String, Object> properties, UUID importId, UUID timestampUuid )
@@ -975,7 +969,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_insertEntity" )
public void insertEntity( EntityRef entityRef ) throws Exception {
String type = entityRef.getType();
@@ -1021,7 +1014,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_getEntityType" )
public String getEntityType( UUID entityId ) throws Exception {
HColumn<String, String> column =
@@ -1044,7 +1036,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_loadPartialEntity" )
public DynamicEntity loadPartialEntity( UUID entityId, String... propertyNames ) throws Exception {
List<HColumn<String, ByteBuffer>> results = null;
@@ -1133,7 +1124,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_getEntities" )
public <A extends Entity> List<A> getEntities(
Collection<UUID> entityIds, Class<A> entityClass ) throws Exception {
@@ -1192,7 +1182,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getPropertyNames" )
public Set<String> getPropertyNames( EntityRef entity ) throws Exception {
Set<String> propertyNames = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
@@ -1215,7 +1204,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryNames" )
public Set<String> getDictionaryNames( EntityRef entity ) throws Exception {
Set<String> dictionaryNames = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
@@ -1239,7 +1227,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValue" )
public Object getDictionaryElementValue( EntityRef entity, String dictionaryName, String elementName )
throws Exception {
@@ -1280,7 +1267,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValues" )
public Map<String, Object> getDictionaryElementValues( EntityRef entity, String dictionaryName,
String... elementNames ) throws Exception {
@@ -1342,7 +1328,6 @@ public class EntityManagerImpl implements EntityManager {
* @throws Exception the exception
*/
@Override
- @Metered( group = "core", name = "EntityManager_getDictionaryAsMap" )
public Map<Object, Object> getDictionaryAsMap( EntityRef entity, String dictionaryName ) throws Exception {
entity = validate( entity );
@@ -1405,7 +1390,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_updateProperties" )
public void updateProperties(
UUID entityId, String type, Map<String, Object> properties ) throws Exception {
@@ -1423,7 +1407,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_deleteEntity" )
public void deleteEntity( UUID entityId, String type ) throws Exception {
logger.info( "deleteEntity {} of application {}", entityId, applicationId );
@@ -1608,7 +1591,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAggregateCounters" )
public Results getAggregateCounters( UUID userId, UUID groupId, UUID queueId, String category,
String counterName, CounterResolution resolution, long start, long finish, boolean pad ) {
@@ -1647,7 +1629,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAggregateCounters_fromQueryObj" )
public Results getAggregateCounters( Query query ) throws Exception {
CounterResolution resolution = query.getResolution();
if ( resolution == null ) {
@@ -1732,7 +1713,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getEntityCounters" )
public Map<String, Long> getEntityCounters( UUID entityId ) throws Exception {
Map<String, Long> counters = new HashMap<String, Long>();
@@ -1755,7 +1735,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_createApplicationCollection" )
public void createApplicationCollection( String entityType ) throws Exception {
Keyspace ko = cass.getApplicationKeyspace( applicationId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index be84176..c4634c4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -81,7 +81,6 @@ import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
import com.google.common.base.Preconditions;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -210,7 +209,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateCollectionIndex")
public IndexUpdate batchUpdateCollectionIndex( IndexUpdate indexUpdate, EntityRef owner, String collectionName )
throws Exception {
@@ -292,7 +290,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollectionIndexes")
public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
// TODO TN, read all buckets here
@@ -391,7 +388,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchAddToCollections")
public Mutator<ByteBuffer> batchAddToCollections( Mutator<ByteBuffer> batch, String ownerType, List<UUID> ownerIds,
String collectionName, Entity entity, UUID timestampUuid )
throws Exception {
@@ -506,7 +502,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchRemoveFromCollection")
public Mutator<ByteBuffer> batchRemoveFromCollection( Mutator<ByteBuffer> batch, String collectionName,
Entity entity, boolean force, UUID timestampUuid )
throws Exception {
@@ -567,7 +562,7 @@ public class RelationManagerImpl implements RelationManager {
if ( !headEntity.getType().equalsIgnoreCase( TYPE_APPLICATION ) && !Schema
.isAssociatedEntityType( entity.getType() ) ) {
- CollectionRef cref = new SimpleCollectionRef( headEntity, collectionName, entity );
+ CollectionRef cref = new SimpleCollectionRef( headEntity, collectionName, entity );
em.delete( new SimpleEntityRef( cref.getType(), cref.getUuid() ) );
}
@@ -575,7 +570,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchDeleteConnectionIndexEntries")
public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries( IndexUpdate indexUpdate, IndexEntry entry,
ConnectionRefImpl connection, UUID[] index_keys )
throws Exception {
@@ -626,7 +620,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchAddConnectionIndexEntries")
public Mutator<ByteBuffer> batchAddConnectionIndexEntries( IndexUpdate indexUpdate, IndexEntry entry,
ConnectionRefImpl connection, UUID[] index_keys ) {
@@ -687,7 +680,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateConnectionIndex")
public IndexUpdate batchUpdateConnectionIndex( IndexUpdate indexUpdate, ConnectionRefImpl connection )
throws Exception {
@@ -733,7 +725,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* addInsertToMutator(batch, EntityCF.SETS, key(connection_id,
* Schema.INDEXES_SET), indexEntry.getKey(), null, false, timestamp); }
- *
+ *
* addInsertToMutator(batch, EntityCF.SETS, key(connection_id,
* Schema.INDEXES_SET), entryName, null, false, timestamp);
*/
@@ -775,7 +767,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsPropertyIndexes")
public IndexUpdate batchUpdateBackwardConnectionsPropertyIndexes( IndexUpdate indexUpdate ) throws Exception {
logger.debug( "batchUpdateBackwordConnectionsPropertyIndexes" );
@@ -851,7 +842,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsDictionaryIndexes")
public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes( IndexUpdate indexUpdate ) throws Exception {
logger.debug( "batchUpdateBackwardConnectionsListIndexes" );
@@ -869,13 +859,12 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchUpdateEntityConnection")
- public Mutator<ByteBuffer> batchUpdateEntityConnection( Mutator<ByteBuffer> batch,
+ public Mutator<ByteBuffer> batchUpdateEntityConnection( Mutator<ByteBuffer> batch,
boolean disconnect, ConnectionRefImpl connection, UUID timestampUuid ) throws Exception {
long timestamp = getTimestampInMicros( timestampUuid );
- Entity connectedEntity = em.get( new SimpleEntityRef(
+ Entity connectedEntity = em.get( new SimpleEntityRef(
connection.getConnectedEntityType(), connection.getConnectedEntityId()) );
if ( connectedEntity == null ) {
@@ -1072,7 +1061,7 @@ public class RelationManagerImpl implements RelationManager {
ConnectionRefImpl loopback = connection.getConnectionToConnectionEntity();
if ( !disconnect ) {
- em.insertEntity( new SimpleEntityRef(
+ em.insertEntity( new SimpleEntityRef(
CONNECTION_ENTITY_CONNECTION_TYPE, loopback.getConnectedEntityId() ) );
}
@@ -1082,7 +1071,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchDisconnect")
public void batchDisconnect( Mutator<ByteBuffer> batch, UUID timestampUuid ) throws Exception {
@@ -1120,7 +1108,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchStartIndexUpdate")
public IndexUpdate batchStartIndexUpdate( Mutator<ByteBuffer> batch, Entity entity, String entryName,
Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
boolean isMultiValue, boolean removeListEntry, boolean fulltextIndexed,
@@ -1250,7 +1237,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchUpdatePropertyIndexes")
public void batchUpdatePropertyIndexes( Mutator<ByteBuffer> batch, String propertyName, Object propertyValue,
boolean entitySchemaHasProperty, boolean noRead, UUID timestampUuid )
throws Exception {
@@ -1416,7 +1402,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
@Override
- @Metered(group = "core", name = "RelationManager_isOwner")
public boolean isCollectionMember( String collectionName, EntityRef entity ) throws Exception {
Keyspace ko = cass.getApplicationKeyspace( applicationId );
@@ -1475,7 +1460,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getOwners")
public Map<String, Map<UUID, Set<String>>> getOwners() throws Exception {
Map<EntityRef, Set<String>> containerEntities = getContainingCollections();
Map<String, Map<UUID, Set<String>>> owners = new LinkedHashMap<String, Map<UUID, Set<String>>>();
@@ -1492,7 +1476,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollections")
public Set<String> getCollections() throws Exception {
Map<String, CollectionInfo> collections = getDefaultSchema().getCollections( headEntity.getType() );
@@ -1505,7 +1488,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollection_start_result")
public Results getCollection( String collectionName, UUID startResult, int count, Level resultsLevel,
boolean reversed ) throws Exception {
// changed intentionally to delegate to search so that behavior is
@@ -1524,7 +1506,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollecitonForQuery")
public Results getCollection( String collectionName, Query query, Level resultsLevel ) throws Exception {
// changed intentionally to delegate to search so that behavior is
@@ -1535,7 +1516,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_addToCollection")
public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception {
Entity itemEntity = em.get( itemRef );
@@ -1566,7 +1546,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_addToCollections")
public Entity addToCollections( List<EntityRef> owners, String collectionName ) throws Exception {
Entity itemEntity = getHeadEntity();
@@ -1600,7 +1579,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createItemInCollection")
public Entity createItemInCollection( String collectionName, String itemType, Map<String, Object> properties )
throws Exception {
@@ -1653,7 +1631,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_removeFromCollection")
public void removeFromCollection( String collectionName, EntityRef itemRef ) throws Exception {
if ( headEntity.getUuid().equals( applicationId ) ) {
@@ -1703,7 +1680,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchRemoveFromContainers")
public void batchRemoveFromContainers( Mutator<ByteBuffer> m, UUID timestampUuid ) throws Exception {
Entity entity = getHeadEntity();
// find all the containing collections
@@ -1720,7 +1696,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_copyRelationships")
public void copyRelationships( String srcRelationName, EntityRef dstEntityRef, String dstRelationName )
throws Exception {
@@ -1757,7 +1732,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_searchCollection")
public Results searchCollection( String collectionName, Query query ) throws Exception {
if ( query == null ) {
@@ -1782,7 +1756,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connection_ref")
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
ConnectionRefImpl connectionImpl = new ConnectionRefImpl( connection );
@@ -1793,7 +1766,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connectionType")
public ConnectionRef createConnection( String connectionType, EntityRef connectedEntityRef ) throws Exception {
headEntity = em.validate( headEntity );
@@ -1808,7 +1780,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_paired_connection_type")
public ConnectionRef createConnection( String pairedConnectionType, EntityRef pairedEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
@@ -1823,7 +1794,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connected_entity_ref")
public ConnectionRef createConnection( ConnectedEntityRef... connections ) throws Exception {
ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connections );
@@ -1835,7 +1805,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_type_entity")
public ConnectionRef connectionRef( String connectionType, EntityRef connectedEntityRef ) throws Exception {
ConnectionRef connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
@@ -1845,7 +1814,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_entity_to_entity")
public ConnectionRef connectionRef( String pairedConnectionType, EntityRef pairedEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
@@ -1858,7 +1826,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_connections")
public ConnectionRef connectionRef( ConnectedEntityRef... connections ) {
ConnectionRef connection = new ConnectionRefImpl( headEntity, connections );
@@ -1868,14 +1835,12 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_deleteConnection")
public void deleteConnection( ConnectionRef connectionRef ) throws Exception {
updateEntityConnection( true, new ConnectionRefImpl( connectionRef ) );
}
@Override
- @Metered(group = "core", name = "RelationManager_getConnectionTypes_entity_id")
public Set<String> getConnectionTypes( UUID connectedEntityId ) throws Exception {
// Add connection type to connections set
// addInsertToMutator(batch, ENTITY_DICTIONARIES,
@@ -1923,7 +1888,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectionTypes")
public Set<String> getConnectionTypes( boolean filterConnection ) throws Exception {
Set<String> connections = cast( em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
if ( connections == null ) {
@@ -1937,7 +1901,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectedEntities")
public Results getConnectedEntities( String connectionType, String connectedEntityType, Level resultsLevel )
throws Exception {
@@ -1976,7 +1939,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectingEntities")
public Results getConnectingEntities( String connectionType, String connectedEntityType,
Level resultsLevel ) throws Exception {
@@ -1985,7 +1947,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectingEntities")
public Results getConnectingEntities(String connectionType,
String entityType, Level level, int count) throws Exception {
return getConnectingEntities(headEntity, connectionType, entityType, level, count );
@@ -2032,7 +1993,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_searchConnectedEntities")
public Results searchConnectedEntities( Query query ) throws Exception {
Preconditions.checkNotNull(query, "Query must not be null");
@@ -2143,7 +2103,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
* persistence.query.ir.WithinNode)
*/
@@ -2165,7 +2125,7 @@ public class RelationManagerImpl implements RelationManager {
@Override
public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
- EntityRef ref = em.getAlias(
+ EntityRef ref = em.getAlias(
headEntity, collection.getType(), nameIdentifierNode.getName() );
if ( ref == null ) {
@@ -2236,7 +2196,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
* persistence.query.ir.WithinNode)
*/
@@ -2324,7 +2284,7 @@ public class RelationManagerImpl implements RelationManager {
@Override
public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
//TODO T.N. USERGRID-1919 actually validate this is connected
- EntityRef ref = em.getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ EntityRef ref = em.getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
connection.getConnectedEntityType(), nameIdentifierNode.getName() );
if ( ref == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index b412df8..ed705b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -27,7 +27,6 @@ import java.util.UUID;
import org.springframework.util.Assert;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -249,7 +248,6 @@ public class ConnectedIndexScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- @Metered( group = "core", name = "IndexBucketScanner_load" )
public Set<HColumn<ByteBuffer, ByteBuffer>> next() {
Set<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index b2ca591..56c651b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -31,7 +31,6 @@ import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -209,7 +208,6 @@ public class IndexBucketScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- @Metered(group = "core", name = "IndexBucketScanner_load")
public NavigableSet<HColumn<ByteBuffer, ByteBuffer>> next() {
NavigableSet<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 25b2258..ee3f9cd 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,7 +109,6 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
@@ -826,12 +825,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- <version>${metrics-version}</version>
- </dependency>
-
- <dependency>
<groupId>com.github.stephenc</groupId>
<artifactId>jamm</artifactId>
<version>0.2.5</version>
@@ -1072,44 +1065,8 @@
<version>${snakeyaml-version}</version>
</dependency>
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics-version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-spring</artifactId>
- <version>${metrics-version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
- </exclusion>
-
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-aop</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-beans</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>jline</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 5a856de..deb11f8 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -259,10 +259,6 @@
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-spring</artifactId>
- </dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
index 27eff28..e4086f9 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
@@ -55,8 +55,6 @@ import org.apache.shiro.authz.UnauthorizedException;
import com.google.common.collect.BiMap;
import com.sun.jersey.api.json.JSONWithPadding;
import com.yammer.metrics.Metrics;
-import com.yammer.metrics.annotation.ExceptionMetered;
-import com.yammer.metrics.annotation.Timed;
import com.yammer.metrics.core.Counter;
import com.yammer.metrics.core.Gauge;
import com.yammer.metrics.core.Histogram;
@@ -297,8 +295,6 @@ public class RootResource extends AbstractContextResource implements MetricProce
public static final String ENTITY_ID_PATH = "{entityId: " + Identifier.UUID_REX + "}";
public static final String EMAIL_PATH = "{email: " + Identifier.EMAIL_REX + "}";
- @Timed(name = "getApplicationByUuids_timer", group = "rest_timers")
- @ExceptionMetered(group = "rest_exceptions", name = "getApplicationByUuids_exceptions")
@Path(ORGANIZATION_ID_PATH+"/"+APPLICATION_ID_PATH)
public ApplicationResource getApplicationByUuids( @PathParam("organizationId") String organizationIdStr,
@PathParam("applicationId") String applicationIdStr )
@@ -324,8 +320,6 @@ public class RootResource extends AbstractContextResource implements MetricProce
}
- @Timed(name = "getOrganizationByName_timer", group = "rest_timers")
- @ExceptionMetered(group = "rest_exceptions", name = "getOrganizationByName_exceptions")
@Path("{organizationName}")
public OrganizationResource getOrganizationByName( @PathParam("organizationName") String organizationName )
throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index fecf069..cf82373 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -149,11 +149,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
[16/50] incubator-usergrid git commit: resolve futures via done method
Posted by to...@apache.org.
resolve futures via done method
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ad33ecfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ad33ecfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ad33ecfd
Branch: refs/heads/USERGRID-405
Commit: ad33ecfd1146a98009c141f1ea6948b9d280f7d7
Parents: 9630fcf
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 16:26:05 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 16:26:05 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexOperationMessage.java | 4 ++++
.../usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java | 1 +
.../usergrid/persistence/index/impl/BufferQueueSQSImpl.java | 2 +-
3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index a7388d6..33b68cd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -126,4 +126,8 @@ public class IndexOperationMessage implements Serializable {
result = 31 * result + deIndexRequests.hashCode();
return result;
}
+
+ public void done() {
+ getFuture().done();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index 1973e5d..6716fd1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -47,6 +47,7 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
messages.offer( operation );
+ operation.done();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 25c2ba6..1b0529f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -141,7 +141,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//signal to SQS
this.queue.sendMessage( identifier );
- operation.getFuture().run();
+ operation.done();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
[22/50] incubator-usergrid git commit: [USERGRID-386] Changed over to
new method em.getAllEntityFromFields. Added unsupported messages to
EntityManagerImpl Changed type of getAllEntityFromFields Added new method to
CpEntityManager that converts a CpEntit
Posted by to...@apache.org.
[USERGRID-386] Changed over to new method em.getAllEntityFromFields.
Added unsupported messages to EntityManagerImpl
Changed type of getAllEntityFromFields
Added new method to CpEntityManager that converts a CpEntity to a regular Entity.
Then adds method to use the getAllEntities method that repairs the entity incase it was deleted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c872b694
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c872b694
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c872b694
Branch: refs/heads/USERGRID-405
Commit: c872b694ed7f9b77220bad81813161ac11b9890f
Parents: a60d6aa
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 11 16:52:04 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 11 16:52:04 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 49 ++++++++++++++------
.../usergrid/persistence/EntityManager.java | 3 +-
.../cassandra/EntityManagerImpl.java | 13 ++++--
.../services/AbstractCollectionService.java | 4 +-
4 files changed, 46 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c872b694/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 4ed37e5..b7272f1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -340,6 +340,17 @@ public class CpEntityManager implements EntityManager {
return entity;
}
+ public Entity convertMvccEntityToEntity( org.apache.usergrid.persistence.model.entity.Entity entity){
+ if(entity == null) {
+ return null;
+ }
+ Class clazz = Schema.getDefaultSchema().getEntityClass( entity.getId().getType() );
+
+ Entity oldFormatEntity = EntityFactory.newEntity(entity.getId().getUuid(),entity.getId().getType(),clazz);
+ oldFormatEntity.setProperties( CpEntityMapUtils.toMap( entity ) );
+
+ return oldFormatEntity;
+ }
@Override
public Entity get( EntityRef entityRef ) throws Exception {
@@ -730,17 +741,33 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Observable<FieldSet> getAllEntityFromFields(String aliasType,String aliasValue ){
- return null;
-// EntityCollectionManager ecm = managerCache.getEntityCollectionManager( aliasType );
-//
-// Schema.
-// return ecm.getAllEntities( aliasValue );
+ public Entity getAllEntityFromFields( String collectionType, String aliasType ){
+ String collName = Schema.defaultCollectionName( collectionType );
+
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
+ getApplicationScope().getApplication(), collName);
+
+ String propertyName = Schema.getDefaultSchema().aliasProperty( collName );
+
+
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ //TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
+
+ StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
+
+ Observable<FieldSet> fieldSetObservable = ecm.getAllEntities( Arrays.<Field>asList( uniqueLookupRepairField) );
+
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+
+ if(fieldSet.isEmpty()) {
+ return null;
+ }
+
+ return convertMvccEntityToEntity( fieldSet.getEntity( uniqueLookupRepairField ).getEntity().get() );
}
@Override
public Entity getEntityByAlias(String collectionType, String aliasType) throws Exception {
-
EntityRef newEntityRef = getAlias(collectionType,aliasType);
if(newEntityRef == null) {
@@ -748,9 +775,6 @@ public class CpEntityManager implements EntityManager {
}
return get( newEntityRef );
-
-
-
}
@@ -2157,11 +2181,6 @@ public class CpEntityManager implements EntityManager {
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
propertyName, propertyValue.toString() ) ).toBlocking() .lastOrDefault( null );
-
- Observable<FieldSet> fieldSetObservable = ecm.getAllEntities( Arrays.<Field>asList( new StringField( propertyName, propertyValue.toString() ) );
-
- FieldSet fieldSet = fieldSetObservable.toBlocking().last();
- fieldSet.
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c872b694/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 4b85c0d..eb6379b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.persistence;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.FieldSet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.index.query.Query;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -719,6 +720,6 @@ public interface EntityManager {
*/
public Health getIndexHealth();
- public Observable<FieldSet> getAllEntityFromFields(String aliasType,String aliasValue );
+ public Entity getAllEntityFromFields( String aliasType, String aliasValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c872b694/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 42a588c..a019e86 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -69,7 +69,6 @@ import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.CounterUtils.AggregateCounterSelection;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
-import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.entities.Event;
import org.apache.usergrid.persistence.entities.Group;
@@ -108,7 +107,6 @@ import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.SliceCounterQuery;
-import rx.Observable;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
@@ -538,7 +536,7 @@ public class EntityManagerImpl implements EntityManager {
/**
* Return all UUIDs that have this unique value
*
- * @param ownerEntityId The entity id that owns this entity collection
+ * @param ownerEntityRef The entity id that owns this entity collection
* @param collectionName The entity collection name
* @param propertyName The name of the unique property
* @param propertyValue The value of the unique property
@@ -1775,6 +1773,12 @@ public class EntityManagerImpl implements EntityManager {
@Override
+ public Entity getEntityByAlias( final String collectionType, final String aliasType ) throws Exception {
+ throw new UnsupportedOperationException("Not supported.");
+ }
+
+
+ @Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
return getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias );
}
@@ -2952,8 +2956,7 @@ public class EntityManagerImpl implements EntityManager {
@Override
- public Observable<FieldSet> getAllEntityFromFields( final String aliasType,
- final String aliasValue ) {
+ public Entity getAllEntityFromFields( final String aliasType, final String aliasValue ) {
throw new UnsupportedOperationException( "Not supported." );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c872b694/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 1117fbc..26eb6b9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -76,7 +76,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- Entity entity = em.getEntityByAlias( getEntityType(), name );
+ Entity entity = em.getAllEntityFromFields( getEntityType(), name );
if ( entity != null ) {
entity = importEntity( request, entity );
}
@@ -148,7 +148,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- EntityRef entity = em.get( getEntityType(), name );
+ EntityRef entity = em.getAlias( getEntityType(), name );
if ( entity == null ) {
logger.info( "miss on entityType: {} with name: {}", getEntityType(), name );
[17/50] incubator-usergrid git commit: Shorted to just get the Entity
by Alias. Corrected spelling Added test that does a get, delete,
get to make sure entities are removed. Refactored entities to getEntityByAlias
Added getEntityByAlias that works based of
Posted by to...@apache.org.
Shorted to just get the Entity by Alias.
Corrected spelling
Added test that does a get,delete,get to make sure entities are removed.
Refactored entities to getEntityByAlias
Added getEntityByAlias that works based off of existing entities.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a60d6aa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a60d6aa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a60d6aa3
Branch: refs/heads/USERGRID-405
Commit: a60d6aa31f089a8b63be7b66f1aed0224837c22b
Parents: d4eb288
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 11 15:36:11 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 11 15:36:11 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 30 ++++++++++++++++++--
.../usergrid/persistence/EntityManager.java | 4 ++-
.../cassandra/EntityManagerImpl.java | 6 ++--
.../usergrid/persistence/EntityManagerIT.java | 24 ++++++++++++++++
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../services/AbstractCollectionService.java | 14 ++++-----
6 files changed, 62 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 4b62fbe..4ed37e5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -730,11 +730,30 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Observable<FieldSet> getAllEntityFromFields(CollectionScope collectionScope,Collection<Field> fields){
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
- return ecm.getAllEntities( fields );
+ public Observable<FieldSet> getAllEntityFromFields(String aliasType,String aliasValue ){
+ return null;
+// EntityCollectionManager ecm = managerCache.getEntityCollectionManager( aliasType );
+//
+// Schema.
+// return ecm.getAllEntities( aliasValue );
+ }
+
+ @Override
+ public Entity getEntityByAlias(String collectionType, String aliasType) throws Exception {
+
+ EntityRef newEntityRef = getAlias(collectionType,aliasType);
+
+ if(newEntityRef == null) {
+ return null;
+ }
+
+ return get( newEntityRef );
+
+
+
}
+
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
@@ -2133,11 +2152,16 @@ public class CpEntityManager implements EntityManager {
getApplicationScope().getApplication(), collectionName);
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+//TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
propertyName, propertyValue.toString() ) ).toBlocking() .lastOrDefault( null );
+ Observable<FieldSet> fieldSetObservable = ecm.getAllEntities( Arrays.<Field>asList( new StringField( propertyName, propertyValue.toString() ) );
+
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+ fieldSet.
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 459032c..4b85c0d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -98,6 +98,8 @@ public interface EntityManager {
public void createApplicationCollection( String entityType ) throws Exception;
+ Entity getEntityByAlias( String collectionType, String aliasType ) throws Exception;
+
public EntityRef getAlias( String aliasType, String alias ) throws Exception;
/**
@@ -717,6 +719,6 @@ public interface EntityManager {
*/
public Health getIndexHealth();
- public Observable<FieldSet> getAllEntityFromFields(CollectionScope collectionScope,Collection<Field> fields);
+ public Observable<FieldSet> getAllEntityFromFields(String aliasType,String aliasValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index adf106a..42a588c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -69,7 +69,6 @@ import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.CounterUtils.AggregateCounterSelection;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
-import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.entities.Event;
@@ -85,7 +84,6 @@ import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.CounterFilterPredicate;
import org.apache.usergrid.persistence.index.query.Query.Level;
-import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.ClassUtils;
import org.apache.usergrid.utils.CompositeUtils;
@@ -2954,8 +2952,8 @@ public class EntityManagerImpl implements EntityManager {
@Override
- public Observable<FieldSet> getAllEntityFromFields( final CollectionScope collectionScope,
- final Collection<Field> fields ) {
+ public Observable<FieldSet> getAllEntityFromFields( final String aliasType,
+ final String aliasValue ) {
throw new UnsupportedOperationException( "Not supported." );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index f8c667d..cd08607 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -44,6 +44,7 @@ import org.apache.usergrid.utils.UUIDUtils;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -578,4 +579,27 @@ public class EntityManagerIT extends AbstractCoreIT {
assertNotNull( em.get( user.getUuid() ));
}
+
+ @Test
+ public void testInvalidNameRepair() throws Exception {
+ LOG.info("EntityManagerIT.testInvalidNameReapir");
+
+ EntityManager em = app.getEntityManager();
+
+ Map<String, Object> properties = new LinkedHashMap<String, Object>();
+ properties.put( "name", "XR-51B" );
+ properties.put( "fuel", "Nutrinox" );
+
+ Entity user = em.create( "robot", properties );
+ assertNotNull( user );
+
+ em.refreshIndex();
+
+ assertNotNull( em.get( user.getUuid() ));
+
+ em.delete( user );
+
+ assertNull( em.get( user.getUuid() ));
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index be95b08..cf84215 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -176,7 +176,7 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
throws ConnectionException {
Preconditions.checkNotNull( fields, "fields are required" );
- Preconditions.checkArgument( fields.size() > 0, "More than 1 field msut be specified" );
+ Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" );
final List<ScopedRowKey<CollectionPrefixedKey<Field>>> keys = new ArrayList<>( fields.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a60d6aa3/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index c3be45f..1117fbc 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -76,11 +76,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- EntityRef entityRef = em.getAlias( getEntityType(), name );
- if ( entityRef == null ) {
- return null;
- }
- Entity entity = em.get( entityRef );
+ Entity entity = em.getEntityByAlias( getEntityType(), name );
if ( entity != null ) {
entity = importEntity( request, entity );
}
@@ -152,7 +148,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- EntityRef entity = em.getAlias( getEntityType(), name );
+ EntityRef entity = em.get( getEntityType(), name );
if ( entity == null ) {
logger.info( "miss on entityType: {} with name: {}", getEntityType(), name );
@@ -178,7 +174,7 @@ public class AbstractCollectionService extends AbstractService {
/*
* Level level = Level.REFS; if (isEmpty(parameters)) {
* level = Level.ALL_PROPERTIES; }
- *
+ *
* Results results = em.searchCollectionForProperty(owner,
* getCollectionName(), null, nameProperty, name, null, null, 1, level);
* EntityRef entity = results.getRef();
@@ -248,9 +244,9 @@ public class AbstractCollectionService extends AbstractService {
}
logger.debug("Limiting collection to " + Query.DEFAULT_LIMIT);
- int count = Query.DEFAULT_LIMIT;
+ int count = Query.DEFAULT_LIMIT;
- Results r = em.getCollection( context.getOwner(), context.getCollectionName(),
+ Results r = em.getCollection( context.getOwner(), context.getCollectionName(),
null, count, Level.ALL_PROPERTIES, isCollectionReversed( context ) );
importEntities( context, r );
[06/50] incubator-usergrid git commit: fix pom
Posted by to...@apache.org.
fix pom
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/89f558ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/89f558ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/89f558ac
Branch: refs/heads/USERGRID-405
Commit: 89f558acdb0c764e1d4f8c7a00cff1b2783d6b2c
Parents: 8ccc3ec
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 11:02:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 11:02:59 2015 -0600
----------------------------------------------------------------------
stack/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/89f558ac/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 73f2cbd..25b2258 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,6 +109,7 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
+ <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
[49/50] incubator-usergrid git commit: Revert "fix for list types
changing from string to int or whatever"
Posted by to...@apache.org.
Revert "fix for list types changing from string to int or whatever"
This reverts commit e5cab76f8b8fc1e4a60ea23274e7a8a0cc79f759.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b09dc435
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b09dc435
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b09dc435
Branch: refs/heads/USERGRID-405
Commit: b09dc43526cd27dcf75fa09dc5c40a288ca30225
Parents: 762e43e
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 17:30:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 17:30:27 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 14 ++-
.../persistence/index/impl/IndexingUtils.java | 109 +++++++++----------
.../persistence/index/impl/EntityIndexTest.java | 38 +------
3 files changed, 70 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 4c32cc1..92312d2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -56,7 +56,15 @@ import com.codahale.metrics.Timer;
import rx.Observable;
import rx.functions.Func1;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -235,7 +243,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put(LIST_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -247,7 +255,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof SetField ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 41a4680..ffd98e9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -34,9 +34,6 @@ public class IndexingUtils {
public static final String STRING_PREFIX = "su_";
public static final String ANALYZED_STRING_PREFIX = "sa_";
- public static final String ARRAY_PREFIX = "ar_";
- public static final String LIST_PREFIX = "li_";
-
public static final String GEO_PREFIX = "go_";
public static final String NUMBER_PREFIX = "nu_";
public static final String BOOLEAN_PREFIX = "bu_";
@@ -156,75 +153,75 @@ public class IndexingUtils {
/** add routing "_routing":{ "required":false, "path":"ug_entityId" **/
.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
- .startArray("dynamic_templates")
+ .startArray("dynamic_templates")
// we need most specific mappings first since it's a stop on match algorithm
- .startObject()
+ .startObject()
- .startObject("entity_id_template" )
- .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping").field("type", "string" )
- .field("index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .startObject( "entity_id_template" )
+ .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
- .startObject()
- .startObject("entity_context_template" )
+ .startObject()
+ .startObject( "entity_context_template" )
.field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
.field( "type", "string" )
- .field("index", "not_analyzed").endObject()
- .endObject()
- .endObject()
-
- .startObject()
- .startObject("entity_version_template" )
- .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
- .field("match_mapping_type", "string" )
- .startObject( "mapping").field("type", "long" )
- .endObject()
- .endObject()
- .endObject()
+ .field( "index", "not_analyzed" ).endObject()
+ .endObject()
+ .endObject()
+
+ .startObject()
+ .startObject( "entity_version_template" )
+ .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
// any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject("template_1" )
+ .startObject()
+ .startObject( "template_1" )
.field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field("match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
.field( "type", "string" )
- .field("index", "analyzed" )
- .endObject()
- .endObject()
+ .field( "index", "analyzed" )
+ .endObject()
+ .endObject()
- .endObject()
+ .endObject()
// all other strings are not analyzed
- .startObject()
- .startObject("template_2" )
+ .startObject()
+ .startObject( "template_2" )
//todo, should be string prefix, remove 2 field mapping
- .field("match", "*" )
- .field("match_mapping_type", "string" )
- .startObject( "mapping" )
- .field("type", "string" )
- .field("index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
// fields names starting with go_ get geo-indexed
- .startObject()
- .startObject("template_3" )
- .field("match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field("type", "geo_point" )
- .endObject()
- .endObject()
- .endObject()
-
- .endArray()
+ .startObject()
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field( "type", "geo_point" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ .endArray()
.endObject();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b09dc435/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 9cb5297..ca9bf79 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -26,7 +26,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
import org.junit.Rule;
@@ -49,6 +51,7 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.jackson.core.type.TypeReference;
@@ -96,7 +99,7 @@ public class EntityIndexTest extends BaseIT {
}
@Test
-// @Ignore("this is a problem i will work on when i can breathe")
+ @Ignore("this is a problem i will work on when i can breathe")
public void testIndexVariations() throws IOException {
Id appId = new SimpleId( "application" );
@@ -116,8 +119,6 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
List<String> list = new ArrayList<>();
list.add("test");
@@ -125,9 +126,6 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
-
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
EntityObject testObj = new EntityObject();
testObj.setField(new StringField("test","testFiedl"));
@@ -135,32 +133,9 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope, entity);
batch.execute().get();
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
- EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
- List<Integer> listint = new ArrayList<>();
- listint.add(0);
- entity.setField(new ArrayField<Integer>("testfield", listint));
- batch.index(indexScope, entity);
- batch.execute().get();
-
- entity = new Entity( entityType );
- entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
- EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
- List<EntityObject> listObj = new ArrayList<>();
- EntityObject listObjField = new EntityObject();
- listObjField.setField(new StringField("testasf","somevalue"));
- listObj.add(listObjField);
- listObjField = new EntityObject();
- listObjField.setField(new IntegerField("testasf",0));
- listObj.add(listObjField);
- entity.setField(new ArrayField<EntityObject>("testfield", listObj));
- batch.index(indexScope, entity);
- batch.execute().get();
-
entityIndex.refresh();
- testQuery(indexScope, searchTypes, entityIndex, "select *", 5);
+ testQueries( indexScope, searchTypes, entityIndex );
}
@Test
@@ -288,7 +263,6 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
//Hilda Youn
-
testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
}
[04/50] incubator-usergrid git commit: Added a master only elastic
search node
Posted by to...@apache.org.
Added a master only elastic search node
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e58597be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e58597be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e58597be
Branch: refs/heads/USERGRID-405
Commit: e58597be99d18ed1f3bfa4387746bdc0254ff122
Parents: b3e42dd
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 10:14:09 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 10:14:09 2015 -0600
----------------------------------------------------------------------
.../dist/init_instance/install_elasticsearch.sh | 12 +-
.../main/groovy/configure_elasticsearch.groovy | 70 ++--
stack/awscluster/ugcluster-cf.json | 326 ++++++++++++++++++-
.../events/EntityDeletedHandler.java | 2 +-
.../events/EntityVersionCreatedHandler.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 26 +-
.../index/impl/EsEntityIndexImpl.java | 166 +++++-----
.../index/impl/EsIndexBufferConsumerImpl.java | 5 +-
8 files changed, 476 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index 06af849..b2018e1 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -42,9 +42,17 @@ echo "vm.swappiness = 0" >> /etc/sysctl.conf
sysctl -p
# No need to do this, elasticsearch nodes are also cassandra nodes
+
cd /usr/share/usergrid/scripts
-groovy registry_register.groovy elasticsearch
-groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
+
+#If we're the master, register ourselves and move on, if we're not, wait for the master to come up
+if [ "$ES_MASTER" = "true" ]; then
+ groovy registry_register.groovy elasticsearch_master
+else
+ groovy registry_register.groovy elasticsearch
+ groovy wait_for_instances.groovy elasticsearch_master 1
+fi
+
# leave room for Cassandra: use about one half of RAM for heap
case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 7bee82c..19c969d 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -18,10 +18,10 @@
-//
-// configure_elasticsearch.groovy
-//
-// Emits Elasticsearch config file based on environment and Elasticsearch node
+//
+// configure_elasticsearch.groovy
+//
+// Emits Elasticsearch config file based on environment and Elasticsearch node
// registry in SimpleDB
//
@@ -29,13 +29,18 @@
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def clusterName = (String)System.getenv().get("ES_CLUSTER_NAME")
+def isMaster = ((String)System.getenv().get("ES_MASTER")).toBoolean()
+
int esNumServers = ((String)System.getenv().get("ES_NUM_SERVERS")).toInteger()
-int quorum = esNumServers/2+1;
+///int quorum = esNumServers/2+1;
+
+//TODO get this from the number of master nodes
+int quorum = 1
NodeRegistry registry = new NodeRegistry();
// build seed list by listing all Elasticsearch nodes found in SimpleDB domain with our stackName
-def selectResult = registry.searchNode('elasticsearch')
+def selectResult = registry.searchNode('elasticsearch_master')
def esnodes = ""
def sep = ""
for (hostname in selectResult) {
@@ -43,13 +48,19 @@ for (hostname in selectResult) {
sep = ","
}
+
+def nodeData = !isMaster
+def nodeMaster = isMaster
+
+
+
def elasticSearchConfig = """
cluster.name: ${clusterName}
discovery.zen.minimum_master_nodes: ${quorum}
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: [${esnodes}]
node:
- name: ${hostName}
+ name: ${hostName}
network:
host: ${hostName}
path:
@@ -65,21 +76,20 @@ es.logger.level: INFO
# Temporarily removing. We don't know better :)
# http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_don_8217_t_touch_these_settings.html#_threadpools
#
-#threadpool:
-# index:
-# type: fixed
-# size: 160
-# queue_size: 0
-# bulk:
-# type: fixed
-# size: 5000
-# size: 16
-# queue_size: 100
-# search:
-# size: 10000
-# size: 48
-# type: fixed
-# queue_size: 100
+threadpool:
+ index:
+ type: fixed
+ size: 160
+ queue_size: 1000
+ bulk:
+ type: fixed
+ size: 160
+ size: 16
+ queue_size: 1000
+ search:
+ size: 320
+ type: fixed
+ queue_size: 1000
action.auto_create_index: false
@@ -108,10 +118,18 @@ cluster.routing.allocation.disk.watermark.low: .97
cluster.routing.allocation.disk.watermark.high: .99
#Set streaming high water marks so reboots don't kill our service
-cluster.routing.allocation.node_concurrent_recoveries: 4
-cluster.routing.allocation.node_initial_primaries_recoveries: 18
-indices.recovery.concurrent_streams: 4
-indices.recovery.max_bytes_per_sec: 40mb
+cluster.routing.allocation.node_concurrent_recoveries: 40
+cluster.routing.allocation.node_initial_primaries_recoveries: 40
+indices.recovery.concurrent_streams: 16
+indices.recovery.max_bytes_per_sec: 300mb
+
+
+##############################
+# Master or data node options
+#############################
+
+node.data: ${nodeData}
+node.master: ${nodeMaster}
###############
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 7926929..0c2a9d2 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -109,6 +109,14 @@
"Default": "6",
"MinValue": "2"
},
+ "ESNumMasterServers": {
+ "Description": "Number of ES master servers to start.",
+ "Type": "Number",
+ "Default": "1",
+ "MinValue": "1"
+ },
+
+
"ESInstanceType": {
"Description": "Instance type for ES servers",
"Type": "String",
@@ -947,7 +955,7 @@
}
}
},
- "ESAutoScalingLaunchConfiguration":{
+ "ESMasterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -979,7 +987,10 @@
"\n",
"export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
"export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "export ES_MASTER=true", "\n",
+
"export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+
"\n",
"export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
"\n",
@@ -1062,7 +1073,7 @@
]
}
},
- "ESAutoScalingGroup": {
+ "ESMasterAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
"Properties": {
@@ -1105,13 +1116,13 @@
}
],
"LaunchConfigurationName": {
- "Ref": "ESAutoScalingLaunchConfiguration"
+ "Ref": "ESMasterAutoScalingLaunchConfiguration"
},
"MinSize": {
- "Ref": "ESNumServers"
+ "Ref": "ESNumMasterServers"
},
"MaxSize": {
- "Ref": "ESNumServers"
+ "Ref": "ESNumMasterServers"
},
"NotificationConfiguration": {
"TopicARN": {
@@ -1251,6 +1262,311 @@
]
}
},
+ "ESAutoScalingLaunchConfiguration":{
+ "Type":"AWS::AutoScaling::LaunchConfiguration",
+ "Properties":{
+ "UserData":{
+ "Fn::Base64":{
+ "Fn::Join":[
+ "",
+ [
+ "#!/bin/bash -ex\n",
+ "# ES NODE STARTUP \n",
+ "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
+ "\n",
+ "mkdir -p /usr/share/usergrid\n",
+ "\n",
+ "# create script that sets our environment variables\n",
+ "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
+ "alias sudo='sudo -E'\n", "\n",
+ "export TYPE=es\n",
+ "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n", "\n",
+ "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
+ "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
+ "\n",
+ "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
+ "export EC2_REGION=", { "Ref":"AWS::Region" },
+ "\n",
+ "export EC2_URL=https://ec2.amazonaws.com/\n",
+ "\n",
+ "export ES_SECURITY_GROUP_NAME=", { "Ref":"ESSecurityGroup" }, "\n",
+ "\n",
+ "\n",
+ "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
+ "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "export ES_MASTER=false","\n",
+ "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+ "\n",
+ "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
+ "\n",
+ "EOF\n",
+ "\n",
+ "# put AWS creds in environment\n",
+ "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
+ "export AWS_ACCESS_KEY=", { "Ref":"ESKey" }, "\n",
+ "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "\n",
+ "# setup s3cmd (will be installed by init script) \n",
+ "cat >/etc/s3cfg <<EOF\n",
+ "access_key=", { "Ref":"ESKey" }, "\n",
+ "secret_key=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] },
+ "\n",
+ "EOF\n",
+ "chmod 644 /etc/s3cfg\n",
+ "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
+ "ln -s /etc/s3cfg ~root/.s3cfg\n",
+ "\n",
+ "# download usergrid and init script bundle from S3\n",
+ "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
+ "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
+ "apt-get update\n",
+ "apt-get -y install s3cmd\n",
+ "cd /usr/share/usergrid\n",
+ "s3cmd --config=/etc/s3cfg get s3://", { "Ref":"ReleaseBucket" }, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "chmod 755 ./init_instance/*.sh\n",
+ "cd ./init_instance\n",
+ "# init as an ES node \n",
+ "sh ./init_es_server.sh\n"
+ ]
+ ]
+ }
+ },
+ "KeyName": {
+ "Ref": "KeyPair"
+ },
+ "ImageId": {
+ "Fn::FindInMap": [
+ "AWSRegionArch2AMI",
+ {
+ "Ref": "AWS::Region"
+ },
+ {
+ "Fn::FindInMap": [
+ "AWSInstanceType2Arch",
+ {
+ "Ref": "CassInstanceType"
+ },
+ "Arch"
+ ]
+ }
+ ]
+ },
+ "InstanceType": {
+ "Ref": "ESInstanceType"
+ },
+ "IamInstanceProfile": {
+ "Ref": "RootInstanceProfile"
+ },
+ "SecurityGroups": [
+ {
+ "Ref": "ESSecurityGroup"
+ }
+ ],
+
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sdb",
+ "VirtualName": "ephemeral0"
+ },
+ {
+ "DeviceName": "/dev/sdc",
+ "VirtualName": "ephemeral1"
+ }
+ ]
+ }
+ },
+ "ESAutoScalingGroup": {
+ "Type": "AWS::AutoScaling::AutoScalingGroup",
+ "Version": "2009-05-15",
+ "Properties": {
+ "AvailabilityZones": [
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ1"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ2"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ3"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ4"
+ ]
+ }
+ ],
+ "LaunchConfigurationName": {
+ "Ref": "ESAutoScalingLaunchConfiguration"
+ },
+ "MinSize": {
+ "Ref": "ESNumServers"
+ },
+ "MaxSize": {
+ "Ref": "ESNumServers"
+ },
+ "NotificationConfiguration": {
+ "TopicARN": {
+ "Ref": "NotificationTopic"
+ },
+ "NotificationTypes": [
+ "autoscaling:EC2_INSTANCE_LAUNCH",
+ "autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
+ "autoscaling:EC2_INSTANCE_TERMINATE",
+ "autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
+ ]
+ }
+ }
+ },
+ "RestAutoScalingLaunchConfiguration":{
+ "Type":"AWS::AutoScaling::LaunchConfiguration",
+ "Properties":{
+ "UserData":{
+ "Fn::Base64":{
+ "Fn::Join":[
+ "",
+ [
+ "#!/bin/bash -ex\n",
+ "# REST SERVER STARTUP \n",
+ "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
+ "\n",
+ "mkdir -p /usr/share/usergrid\n",
+ "\n",
+ "# create script that sets our environment variables\n",
+ "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
+ "alias sudo='sudo -E'\n",
+ "\n",
+ "export TYPE=rest\n",
+ "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n",
+ "export YOURKIT=", { "Ref":"InstallYourkit" }, "\n",
+ "export DNS_NAME=", { "Ref":"DnsSubDomain" }, "\n",
+ "export DNS_DOMAIN=", { "Ref":"DnsDomain" }, "\n",
+ "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
+ "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
+ "export ELB_NAME=", { "Ref":"RestElasticLoadBalancer" }, "\n",
+ "\n",
+ "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
+ "export EC2_REGION=", { "Ref":"AWS::Region" }, "\n",
+ "export EC2_URL=https://ec2.amazonaws.com/\n", "\n",
+ "export REST_SECURITY_GROUP_NAME=", { "Ref":"RestSecurityGroup" }, "\n",
+ "export DB_SECURITY_GROUP_NAME=", { "Ref":"CassSecurityGroup" }, "\n",
+ "\n",
+ "export CASSANDRA_CLUSTER_NAME=", { "Ref":"CassClusterName" }, "\n",
+ "export CASSANDRA_KEYSPACE_NAME=usergrid", "\n",
+ "export CASSANDRA_NUM_SERVERS=", { "Ref":"CassNumServers" }, "\n",
+ "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+ "export TOMCAT_NUM_SERVERS=", { "Ref":"RestMinServers" }, "\n",
+ "\n",
+ "export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor" }, "\n",
+ "\n",
+ "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency" }, "\n",
+ "\n",
+ "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
+ "\n",
+
+ "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
+ "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "\n",
+ "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
+ "\n",
+ "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
+ "\n",
+ "export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
+ "export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
+ "\n",
+ "EOF\n",
+ "\n",
+ "# put AWS creds in environment\n",
+ "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
+ "export AWS_ACCESS_KEY=", { "Ref":"RestKey" }, "\n",
+ "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "\n",
+ "# setup s3cmd (will be installed by init script) \n",
+ "cat >/etc/s3cfg <<EOF\n",
+ "access_key=", { "Ref":"RestKey" }, "\n",
+ "secret_key=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "chmod 644 /etc/s3cfg\n",
+ "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
+ "ln -s /etc/s3cfg ~root/.s3cfg\n",
+ "\n",
+ "# download usergrid and init script bundle from S3\n",
+ "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
+ "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
+ "apt-get update\n",
+ "apt-get -y install s3cmd\n",
+ "cd /usr/share/usergrid\n",
+ "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/ROOT.war\n",
+ "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "mv ROOT.war webapps/ROOT.war\n",
+ "chmod 755 ./init_instance/*.sh\n",
+ "cd ./init_instance\n",
+ "# Init as a REST intance \n",
+ "sh ./init_rest_server.sh\n"
+ ]
+ ]
+ }
+ },
+ "KeyName":{
+ "Ref":"KeyPair"
+ },
+ "ImageId":{
+ "Fn::FindInMap":[
+ "AWSRegionArch2AMI",
+ {
+ "Ref":"AWS::Region"
+ },
+ {
+ "Fn::FindInMap":[
+ "AWSInstanceType2Arch",
+ {
+ "Ref":"RestInstanceType"
+ },
+ "Arch"
+ ]
+ }
+ ]
+ },
+ "InstanceType":{
+ "Ref":"RestInstanceType"
+ },
+ "IamInstanceProfile":{
+ "Ref":"RootInstanceProfile"
+ },
+ "SecurityGroups":[
+ {
+ "Ref":"RestSecurityGroup"
+ }
+ ]
+ }
+ },
"RestAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index ad4ed32..33cc988 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -63,6 +63,6 @@ public class EntityDeletedHandler implements EntityDeleted {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- ei.deleteAllVersionsOfEntity( entityId );
+// ei.deleteAllVersionsOfEntity( entityId );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 3b7163c..590df61 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -64,6 +64,6 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
+// ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 3edde4d..aa2bf8e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -74,19 +74,19 @@ public interface EntityIndex {
*/
public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id);
- /**
- * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
- * effectively removing all versions of an entity from all index scopes
- * @param entityId The entityId to remove
- */
- public Future deleteAllVersionsOfEntity(final Id entityId );
-
- /**
- * Takes all the previous versions of the current entity and deletes all previous versions
- * @param id The id to remove
- * @param version The max version to retain
- */
- public Future deletePreviousVersions(final Id id, final UUID version);
+// /**
+// * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
+// * effectively removing all versions of an entity from all index scopes
+// * @param entityId The entityId to remove
+// */
+// public Future deleteAllVersionsOfEntity(final Id entityId );
+//
+// /**
+// * Takes all the previous versions of the current entity and deletes all previous versions
+// * @param id The id to remove
+// * @param version The max version to retain
+// */
+// public Future deletePreviousVersions(final Id id, final UUID version);
/**
* Refresh the index.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 99d2b84..99643da 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -662,89 +662,89 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
- @Override
- public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
-
- String idString = IndexingUtils.idString(entityId).toLowerCase();
-
- final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
-
- //Added For Graphite Metrics
- final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
- final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
- final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
- .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
-
- response.addListener( new ActionListener<DeleteByQueryResponse>() {
-
- @Override
- public void onResponse( DeleteByQueryResponse response) {
- timeDeleteAllVersions.stop();
- logger
- .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
- entityId.getUuid(), response.status().toString() );
-
- checkDeleteByQueryResponse(tqb, response);
- }
-
-
- @Override
- public void onFailure( Throwable e ) {
- timeDeleteAllVersions.stop();
- logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
- entityId.getUuid(), e);
-
-
- }
- });
- timeDeleteAllVersionsFuture.stop();
- return response;
- }
-
-
- @Override
- public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
-
- String idString = IndexingUtils.idString( entityId ).toLowerCase();
-
- final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
- FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
- );
-
- //Added For Graphite Metrics
- //Checks the time from the execute to the response below
- final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
- final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
- final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
- .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
-
- //Added For Graphite Metrics
- response.addListener(new ActionListener<DeleteByQueryResponse>() {
- @Override
- public void onResponse(DeleteByQueryResponse response) {
- timeDeletePreviousVersions.stop();
- //error message needs to be retooled so that it describes the entity more throughly
- logger
- .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
- entityId.getType(), entityId.getUuid(), version, response.status().toString());
-
- checkDeleteByQueryResponse( fqb, response );
- }
-
-
- @Override
- public void onFailure( Throwable e ) {
- timeDeletePreviousVersions.stop();
- logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
- entityId.getUuid(), e );
- }
- } );
-
- timeDeletePreviousVersionFuture.stop();
-
- return response;
- }
+// @Override
+// public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
+// String idString = IndexingUtils.idString(entityId).toLowerCase();
+//
+// final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
+//
+// //Added For Graphite Metrics
+// final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
+// final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
+//
+// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
+// .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
+//
+// response.addListener( new ActionListener<DeleteByQueryResponse>() {
+//
+// @Override
+// public void onResponse( DeleteByQueryResponse response) {
+// timeDeleteAllVersions.stop();
+// logger
+// .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+// entityId.getUuid(), response.status().toString() );
+//
+// checkDeleteByQueryResponse(tqb, response);
+// }
+//
+//
+// @Override
+// public void onFailure( Throwable e ) {
+// timeDeleteAllVersions.stop();
+// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+// entityId.getUuid(), e);
+//
+//
+// }
+// });
+// timeDeleteAllVersionsFuture.stop();
+// return response;
+// }
+//
+//
+// @Override
+// public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
+//
+// String idString = IndexingUtils.idString( entityId ).toLowerCase();
+//
+// final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
+// QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
+// FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
+// );
+//
+// //Added For Graphite Metrics
+// //Checks the time from the execute to the response below
+// final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
+// final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
+// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
+// .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
+//
+// //Added For Graphite Metrics
+// response.addListener(new ActionListener<DeleteByQueryResponse>() {
+// @Override
+// public void onResponse(DeleteByQueryResponse response) {
+// timeDeletePreviousVersions.stop();
+// //error message needs to be retooled so that it describes the entity more throughly
+// logger
+// .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
+// entityId.getType(), entityId.getUuid(), version, response.status().toString());
+//
+// checkDeleteByQueryResponse( fqb, response );
+// }
+//
+//
+// @Override
+// public void onFailure( Throwable e ) {
+// timeDeletePreviousVersions.stop();
+// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+// entityId.getUuid(), e );
+// }
+// } );
+//
+// timeDeletePreviousVersionFuture.stop();
+//
+// return response;
+// }
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 3fc3e77..2762c18 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -72,8 +72,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//the actively running subscription
private Subscription subscription;
- private Observable<List<IndexOperationMessage>> consumer;
-
private Object mutex = new Object();
@Inject
@@ -203,6 +201,9 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ indexSizeCounter.inc(indexOperationMessage.getDeIndexRequests().size());
+ indexSizeCounter.inc(indexOperationMessage.getIndexRequests().size());
+
return Observable.merge( index, deIndex );
}
} )
[24/50] incubator-usergrid git commit: Added the ability to specify
the number of workers
Posted by to...@apache.org.
Added the ability to specify the number of workers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9aec790b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9aec790b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9aec790b
Branch: refs/heads/USERGRID-405
Commit: 9aec790b3fd8486ce6dba67125f94a5d912e8596
Parents: d81dfaa
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 18:27:16 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 18:27:16 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 9 ++++
.../index/impl/EsIndexBufferConsumerImpl.java | 55 +++++++++++++++-----
.../index/guice/TestIndexModule.java | 4 +-
3 files changed, 52 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index cde86fd..445789f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -81,6 +81,11 @@ public interface IndexFig extends GuicyFig {
*/
public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+ /**
+ * The number of worker threads to consume from the queue
+ */
+ public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -181,4 +186,8 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_QUEUE_READ_TIMEOUT )
int getIndexQueueTimeout();
+ @Default("2")
+ @Key( ELASTICSEARCH_WORKER_COUNT )
+ int getWorkerCount();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 862b1ae..836ec3d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -49,9 +49,12 @@ import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* Consumer for IndexOperationMessages
@@ -69,15 +72,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Meter flushMeter;
private final Timer produceTimer;
private final BufferQueue bufferQueue;
+ private final IndexFig indexFig;
+ private final AtomicLong counter = new AtomicLong( );
//the actively running subscription
- private Subscription subscription;
+ private List<Subscription> subscriptions;
private Object mutex = new Object();
@Inject
- public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider
- provider, final MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
+ public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
+ metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
+
this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
@@ -86,15 +92,42 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
this.client = provider.getClient();
this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
this.bufferQueue = bufferQueue;
+ this.indexFig = indexFig;
-
+ subscriptions = new ArrayList<>( indexFig.getWorkerCount() );
//batch up sets of some size and send them in batch
start();
}
+ /**
+ * Loop throught and start the workers
+ */
public void start() {
+ final int count = indexFig.getWorkerCount();
+
+ for(int i = 0; i < count; i ++){
+ startWorker();
+ }
+ }
+
+
+ /**
+ * Stop the workers
+ */
+ public void stop() {
+ synchronized ( mutex ) {
+ //stop consuming
+
+ for(final Subscription subscription: subscriptions){
+ subscription.unsubscribe();
+ }
+ }
+ }
+
+
+ private void startWorker(){
synchronized ( mutex) {
final AtomicInteger countFail = new AtomicInteger();
@@ -104,7 +137,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
//name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
+ Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
List<IndexOperationMessage> drainList;
do {
@@ -168,20 +201,14 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
} );
//start in the background
- subscription = consumer.subscribe();
- }
- }
+ final Subscription subscription = consumer.subscribe();
- public void stop() {
- synchronized ( mutex ) {
- //stop consuming
- if(subscription != null) {
- subscription.unsubscribe();
- }
+ subscriptions.add(subscription );
}
}
+
/**
* Execute the request, check for errors, then re-init the batch for future use
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7d7a18d..57c2fab 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -38,8 +38,8 @@ public class TestIndexModule extends TestModule {
install( new IndexModule() {
@Override
public void wireBufferQueue() {
-// bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
- bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
+ bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
+// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
}
[18/50] incubator-usergrid git commit: Fixed naming convention issue
Posted by to...@apache.org.
Fixed naming convention issue
Fixes blocking on cleanup (not necessary)
Fixes index creation when alias is cached
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c5943465
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c5943465
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c5943465
Branch: refs/heads/USERGRID-405
Commit: c594346558b9624c05f62096f8d83a0a91ce8af5
Parents: 9630fcf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:02:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:02:58 2015 -0600
----------------------------------------------------------------------
.../results/FilteringLoader.java | 2 +-
.../index/impl/BufferQueueSQSImpl.java | 14 +-
.../index/impl/EsEntityIndexBatchImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 14 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 19 +--
.../persistence/index/impl/EsIndexCache.java | 138 +++++++++++--------
.../queue/impl/SQSQueueManagerImpl.java | 10 +-
7 files changed, 124 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 1afd76b..2cd9fdc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -233,7 +233,7 @@ public class FilteringLoader implements ResultsLoader {
@Override
public void postProcess() {
- this.indexBatch.execute().get();
+ this.indexBatch.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 25c2ba6..3cace11 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -127,6 +127,12 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
+ //no op
+ if(operation.isEmpty()){
+ operation.getFuture().done();
+ return;
+ }
+
final Timer.Context timer = this.writeTimer.time();
this.writeMeter.mark();
@@ -141,7 +147,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//signal to SQS
this.queue.sendMessage( identifier );
- operation.getFuture().run();
+ operation.getFuture().done();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -184,7 +190,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
//look up the values
- final Map<String, String> values = mapManager.getStrings( mapEntries );
+ final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
//load them into our response
@@ -193,7 +199,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final String key = getMessageKey( message );
//now see if the key was there
- final String payload = values.get( key );
+ final String payload = storedCommands.get( key );
//the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
// a DLQ
@@ -242,7 +248,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
- final String key = getMessageKey(sqsIndexOperationMessage.getMessage());
+ final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
//remove it from the map
mapManager.delete( key );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b63dfe6..8481dab 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -197,7 +197,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* No-op, just disregard it
*/
if(tempContainer.isEmpty()){
- return tempContainer.getFuture();
+ final BetterFuture<?> future = tempContainer.getFuture();
+ future.done();
+ return future;
}
return indexBatchBufferProducer.put(tempContainer);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 99643da..fa50734 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -209,7 +209,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- String[] indexes = getIndexes(AliasType.Write);
+ String[] indexes = getIndexesFromEs(AliasType.Write);
if(indexes == null || indexes.length==0) {
addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
}
@@ -336,6 +336,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
/**
+ * Get our index info from ES, but clear our cache first
+ * @param aliasType
+ * @return
+ */
+ public String[] getIndexesFromEs(final AliasType aliasType){
+ aliasCache.invalidate( alias );
+ return getIndexes( aliasType );
+ }
+
+
+
+ /**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
* http://s.apache.org/index-missing-exception
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 2762c18..862b1ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -46,6 +46,7 @@ import rx.Subscription;
import rx.functions.Action1;
import rx.functions.Action2;
import rx.functions.Func1;
+import rx.functions.Func2;
import rx.schedulers.Schedulers;
import java.util.List;
@@ -191,27 +192,27 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
//process and flatten all the messages to builder requests
- Observable<IndexOperationMessage> flattenMessages = Observable.from( operationMessages );
-
-
//batch shard operations into a bulk request
- flattenMessages.flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
+ Observable.from( operationMessages ).flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
@Override
public Observable<BatchRequest> call( final IndexOperationMessage indexOperationMessage ) {
final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
- final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ final Observable<DeIndexRequest> deIndex =
+ Observable.from( indexOperationMessage.getDeIndexRequests() );
- indexSizeCounter.inc(indexOperationMessage.getDeIndexRequests().size());
- indexSizeCounter.inc(indexOperationMessage.getIndexRequests().size());
+ indexSizeCounter.dec( indexOperationMessage.getDeIndexRequests().size() );
+ indexSizeCounter.dec( indexOperationMessage.getIndexRequests().size() );
return Observable.merge( index, deIndex );
}
} )
//collection all the operations into a single stream
- .collect( initRequest(), new Action2<BulkRequestBuilder, BatchRequest>() {
+ .reduce( initRequest(), new Func2<BulkRequestBuilder, BatchRequest, BulkRequestBuilder>() {
@Override
- public void call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
batchRequest.doOperation( client, bulkRequestBuilder );
+
+ return bulkRequestBuilder;
}
} )
//send the request off to ES
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index 0c07a34..ef518dd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -20,6 +20,24 @@
package org.apache.usergrid.persistence.index.impl;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -29,21 +47,6 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
/**
@@ -52,65 +55,88 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class EsIndexCache {
- private static final Logger logger = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<String, String[]> aliasIndexCache;
+ private EsProvider provider;
+
@Inject
- public EsIndexCache(final EsProvider provider, final IndexFig indexFig) {
-
- this.refreshExecutors = MoreExecutors
- .listeningDecorator(Executors.newScheduledThreadPool(indexFig.getIndexCacheMaxWorkers()));
-
- aliasIndexCache = CacheBuilder.newBuilder().maximumSize(1000)
- .refreshAfterWrite(5,TimeUnit.MINUTES)
- .build(new CacheLoader<String, String[]>() {
- @Override
- public ListenableFuture<String[]> reload(final String key, String[] oldValue) throws Exception {
- ListenableFutureTask<String[]> task = ListenableFutureTask.create( new Callable<String[]>() {
- public String[] call() {
- return load( key );
- }
- } );
- refreshExecutors.execute(task);
- return task;
- }
-
- @Override
- public String[] load(final String aliasName) {
- final AdminClient adminClient = provider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
- adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
- return aliasMap.keys().toArray(String.class);
- }
- }) ;
+ public EsIndexCache( final EsProvider provider, final IndexFig indexFig ) {
+
+ this.refreshExecutors =
+ MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
+
+ this.provider = provider;
+
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
+ .build( new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload( final String key,
+ String[] oldValue )
+ throws Exception {
+ ListenableFutureTask<String[]> task =
+ ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute( task );
+ return task;
+ }
+
+
+ @Override
+ public String[] load( final String aliasName ) {
+ return getIndexesFromEs(aliasName);
+ }
+ } );
}
-
+
/**
* Get indexes for an alias
*/
- public String[] getIndexes(IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
String[] indexes;
try {
- indexes = aliasIndexCache.get(aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias());
- } catch (ExecutionException ee) {
- logger.error("Failed to retreive indexes", ee);
- throw new RuntimeException(ee);
+ indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
+ }
+ catch ( ExecutionException ee ) {
+ logger.error( "Failed to retreive indexes", ee );
+ throw new RuntimeException( ee );
}
return indexes;
}
-
+
+
+ private String[] getIndexesFromEs(final String aliasName){
+ final AdminClient adminClient = this.provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
+ adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
+ return aliasMap.keys().toArray( String.class );
+ }
+
+
/**
- * clean up cache
+ * Get the name of the alias to use
+ * @param alias
+ * @param aliasType
+ * @return
*/
- public void invalidate(IndexIdentifier.IndexAlias alias){
- aliasIndexCache.invalidate(alias.getWriteAlias());
- aliasIndexCache.invalidate(alias.getReadAlias());
-
+ private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
}
+
+ /**
+ * clean up cache
+ */
+ public void invalidate( IndexIdentifier.IndexAlias alias ) {
+ aliasIndexCache.invalidate( alias.getWriteAlias() );
+ aliasIndexCache.invalidate( alias.getReadAlias() );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index a78fc80..1fbd9b6 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -33,7 +33,6 @@ import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueMessage;
import org.apache.usergrid.persistence.queue.QueueScope;
-import com.amazonaws.AbortedException;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
@@ -76,15 +75,16 @@ public class SQSQueueManagerImpl implements QueueManager {
.maximumSize( 1000 )
.build( new CacheLoader<String, Queue>() {
@Override
- public Queue load( String queueLoader ) throws Exception {
+ public Queue load( String queueName ) throws Exception {
//the amazon client is not thread safe, we need to create one per queue
Queue queue = null;
try {
- GetQueueUrlResult result = sqs.getQueueUrl( queueLoader );
+ GetQueueUrlResult result = sqs.getQueueUrl( queueName );
queue = new Queue( result.getQueueUrl() );
}catch ( QueueDoesNotExistException queueDoesNotExistException ) {
//no op, swallow
+ LOG.error( "Queue {} does not exist, creating", queueName );
}
catch ( Exception e ) {
@@ -92,7 +92,7 @@ public class SQSQueueManagerImpl implements QueueManager {
throw e;
}
if ( queue == null ) {
- CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueLoader );
+ CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueName );
CreateQueueResult result = sqs.createQueue( createQueueRequest );
String url = result.getQueueUrl();
queue = new Queue( url );
@@ -124,7 +124,7 @@ public class SQSQueueManagerImpl implements QueueManager {
private String getName() {
- String name = scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
+ String name = fig.getPrefix() + "_" + scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
return name;
}
[23/50] incubator-usergrid git commit: change metrics prefix
Posted by to...@apache.org.
change metrics prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e8dc17de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e8dc17de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e8dc17de
Branch: refs/heads/USERGRID-405
Commit: e8dc17de71638ec6f62412a173cc3e161046c4c2
Parents: d81dfaa
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 17:59:07 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 17:59:07 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/core/metrics/MetricsFactoryImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e8dc17de/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 773828d..6d0881b 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -49,7 +49,7 @@ public class MetricsFactoryImpl implements MetricsFactory {
if(!metricsHost.equals("false")) {
Graphite graphite = new Graphite(new InetSocketAddress(metricsHost, 2003));
graphiteReporter = GraphiteReporter.forRegistry(registry)
- .prefixedWith("notifications")
+ .prefixedWith("usergrid-metrics")
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(MetricFilter.ALL)
[20/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/77542c6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/77542c6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/77542c6b
Branch: refs/heads/USERGRID-405
Commit: 77542c6b1bd96f402b33857b453431689ec5b085
Parents: fa0705e ad33ecf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:05:47 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:05:47 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexOperationMessage.java | 4 ++++
.../usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java | 1 +
.../usergrid/persistence/index/impl/BufferQueueSQSImpl.java | 2 +-
3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/77542c6b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
[32/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fde59b70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fde59b70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fde59b70
Branch: refs/heads/USERGRID-405
Commit: fde59b709ca242836dccd78e9af6d711a155a130
Parents: 9e6e36d 19c6ad0
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 12 17:29:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 12 17:29:59 2015 -0600
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 8 +-
stack/awscluster/ugcluster-cf.json | 150 ++-----------------
.../usergrid/persistence/index/IndexFig.java | 9 ++
.../index/impl/EsIndexBufferConsumerImpl.java | 55 +++++--
.../index/guice/TestIndexModule.java | 4 +-
5 files changed, 75 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
[30/50] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-396
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-396
# By Todd Nine (13) and others
# Via Todd Nine (3) and others
* 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid: (32 commits)
Fixes ttl on map entry keys
Addes Cache to MapMangerFactorImpl for efficiency
Added a timeout to the map module
change timer for buffer consumer
change timer for buffer consumer
Moved remaining CpEntityManager timers into the constructor.
Changed log level to warn to catch issues with version mismatch in logging
Changes index declaration to be explicit on the index
Added operational logging to tasks
Moved add alias and test new index into try so we don't keep moving the index alias
Fixes merge error
move timers up
move timers up
move timers up
Adds put test to gatling
Fixes lookup cache bug
changing buffer impl to queue
changing buffer impl to queue
Added caching to org/app lookup
Upgraded plexus utils and fixes build concurrency issues with surefire
...
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c45a3ed0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c45a3ed0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c45a3ed0
Branch: refs/heads/USERGRID-405
Commit: c45a3ed0b08ac3b175d5e3cf214b8352c271ef77
Parents: 4c0102e 06e7ad6
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 12 15:41:03 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 12 15:41:03 2015 -0700
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 ++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
stack/core/pom.xml | 7 +-
.../corepersistence/CpEntityManager.java | 140 +++++++++-
.../corepersistence/CpEntityManagerFactory.java | 156 ++++--------
.../corepersistence/CpRelationManager.java | 39 ++-
.../corepersistence/OrgApplicationCache.java | 67 +++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../core/metrics/MetricsFactoryImpl.java | 3 +-
.../usergrid/persistence/map/MapManager.java | 8 +
.../persistence/map/MapManagerFactory.java | 2 +
.../persistence/map/guice/MapModule.java | 6 +-
.../map/impl/MapManagerFactoryImpl.java | 67 +++++
.../persistence/map/impl/MapManagerImpl.java | 6 +
.../persistence/map/impl/MapSerialization.java | 20 +-
.../map/impl/MapSerializationImpl.java | 106 ++++++--
.../persistence/map/MapManagerTest.java | 49 +++-
stack/corepersistence/queryindex/pom.xml | 16 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../persistence/index/guice/IndexModule.java | 11 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 85 +++++++
.../index/impl/EsEntityIndexImpl.java | 253 +++++++++++++++----
.../index/impl/EsIndexBufferConsumerImpl.java | 48 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 3 -
.../persistence/index/utils/StringUtils.java | 126 +--------
.../persistence/index/utils/UUIDUtils.java | 4 -
.../persistence/index/impl/EntityIndexTest.java | 101 +++++++-
.../datagenerators/EntityDataGenerator.scala | 25 ++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 ++++++
stack/pom.xml | 24 +-
stack/rest/pom.xml | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
stack/services/pom.xml | 7 +-
.../services/notifications/wns/WNSAdapter.java | 39 +++
.../resources/usergrid-services-context.xml | 8 +-
.../usergrid/management/OrganizationIT.java | 2 +-
53 files changed, 1491 insertions(+), 411 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c45a3ed0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
[43/50] incubator-usergrid git commit: Merge remote-tracking branch
'origin/USERGRID-466-change-write-consistency' into USERGRID-466
Posted by to...@apache.org.
Merge remote-tracking branch 'origin/USERGRID-466-change-write-consistency' into USERGRID-466
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/205e0e0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/205e0e0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/205e0e0c
Branch: refs/heads/USERGRID-405
Commit: 205e0e0c03c176623fe9fa2cc18e6a32878838b5
Parents: 2d6ae36 45271d7
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 13:56:33 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 13:56:33 2015 -0600
----------------------------------------------------------------------
.../collection/EntityDeletedFactory.java | 34 -----
.../collection/EntityVersionCleanupFactory.java | 35 -----
.../collection/EntityVersionCreatedFactory.java | 31 -----
.../impl/EntityVersionTaskFactory.java | 65 ++++++++++
.../mvcc/stage/write/WriteUniqueVerify.java | 130 ++++++++++++-------
.../UniqueValueSerializationStrategy.java | 21 ++-
.../UniqueValueSerializationStrategyImpl.java | 19 ++-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 7 +-
.../astyanax/MultiRowColumnIteratorTest.java | 7 +-
.../index/impl/EsEntityIndexImpl.java | 31 +++--
15 files changed, 238 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
index 3c673ee,3c673ee..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
+++ /dev/null
@@@ -1,34 -1,34 +1,0 @@@
--/*
-- *
-- * * Licensed to the Apache Software Foundation (ASF) under one or more
-- * * contributor license agreements. The ASF licenses this file to You
-- * * under the Apache License, Version 2.0 (the "License"); you may not
-- * * use this file except in compliance with the License.
-- * * You may obtain a copy of the License at
-- * *
-- * * http://www.apache.org/licenses/LICENSE-2.0
-- * *
-- * * Unless required by applicable law or agreed to in writing, software
-- * * distributed under the License is distributed on an "AS IS" BASIS,
-- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * * See the License for the specific language governing permissions and
-- * * limitations under the License. For additional information regarding
-- * * copyright in this work, please see the NOTICE file in the top level
-- * * directory of this distribution.
-- *
-- */
--
--package org.apache.usergrid.persistence.collection;
--
--import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--import java.util.UUID;
--
--/**
-- * Creates EntityDeletedTask instances
-- */
--public interface EntityDeletedFactory {
-- public EntityDeletedTask getTask(final CollectionScope collectionScope, final Id entityId, final UUID version );
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
index 2232f00,2232f00..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
+++ /dev/null
@@@ -1,35 -1,35 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. The ASF licenses this file to You
-- * under the Apache License, Version 2.0 (the "License"); you may not
-- * use this file except in compliance with the License.
-- * You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License. For additional information regarding
-- * copyright in this work, please see the NOTICE file in the top level
-- * directory of this distribution.
-- */
--package org.apache.usergrid.persistence.collection;
--
--
--
--import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--import java.util.UUID;
--
--
--public interface EntityVersionCleanupFactory {
--
-- public EntityVersionCleanupTask getTask(
-- final CollectionScope scope,
-- final Id entityId,
-- final UUID version );
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
index 4248d42,4248d42..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
+++ /dev/null
@@@ -1,31 -1,31 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing,
-- * software distributed under the License is distributed on an
-- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-- * KIND, either express or implied. See the License for the
-- * specific language governing permissions and limitations
-- * under the License.
-- */
--package org.apache.usergrid.persistence.collection;
--
--
--import java.util.UUID;
--
--import org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask;
--import org.apache.usergrid.persistence.model.entity.Entity;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--public interface EntityVersionCreatedFactory {
-- public EntityVersionCreatedTask getTask( final CollectionScope scope, final Entity entity);
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
index 0000000,0000000..b41e668
new file mode 100644
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
@@@ -1,0 -1,0 +1,65 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. The ASF licenses this file to You
++ * under the Apache License, Version 2.0 (the "License"); you may not
++ * use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License. For additional information regarding
++ * copyright in this work, please see the NOTICE file in the top level
++ * directory of this distribution.
++ */
++package org.apache.usergrid.persistence.collection.impl;
++
++
++
++import org.apache.usergrid.persistence.collection.CollectionScope;
++import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
++import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
++import org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask;
++import org.apache.usergrid.persistence.model.entity.Entity;
++import org.apache.usergrid.persistence.model.entity.Id;
++
++import java.util.UUID;
++
++
++public interface EntityVersionTaskFactory {
++
++ /**
++ * Get a task for cleaning up latent entity data. If includeVersion = true, the passed version will be cleaned up as well
++ * Otherwise this is a V-1 operation
++ *
++ * @param scope
++ * @param entityId
++ * @param version
++ * @param includeVersion
++ * @return
++ */
++ public EntityVersionCleanupTask getCleanupTask( final CollectionScope scope, final Id entityId, final UUID version,
++ final boolean includeVersion );
++
++ /**
++ * Get an entityVersionCreatedTask
++ * @param scope
++ * @param entity
++ * @return
++ */
++ public EntityVersionCreatedTask getCreatedTask( final CollectionScope scope, final Entity entity );
++
++ /**
++ * Get an entity deleted task
++ * @param collectionScope
++ * @param entityId
++ * @param version
++ * @return
++ */
++ public EntityDeletedTask getDeleteTask( final CollectionScope collectionScope, final Id entityId,
++ final UUID version );
++
++}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8be044f,9343d39..3a11ec7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -169,26 -173,36 +172,36 @@@ public class EsEntityIndexImpl implemen
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
- this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.timer" );
this.removeAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "remove.alias.timer" );
this.addReadAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.read.alias.timer" );
this.addWriteAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.write.alias.timer" );
this.mappingTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" );
+ .getTimer( EsEntityIndexImpl.class, "create.mapping.timer" );
this.refreshTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" );
+ .getTimer( EsEntityIndexImpl.class, "refresh.timer" );
this.searchTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
+ this.allVersionsTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
+ this.deletePreviousTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
+ this.allVersionsTimerFuture = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
+ this.deletePreviousTimerFuture = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
+
+ this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
+
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
mapManager = mapManagerFactory.createMapManager(mapScope);
[44/50] incubator-usergrid git commit: Moved task generation into
single factory to clean up code
Posted by to...@apache.org.
Moved task generation into single factory to clean up code
Fixes repair by version
Fixes incorrect task invocation during writes
Fixes swallowed exception bug
Adds on error processing to QueueConsumer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f20dfd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f20dfd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f20dfd3c
Branch: refs/heads/USERGRID-405
Commit: f20dfd3cbee1f3abb09ae04036b9e747d2485397
Parents: 205e0e0
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 13:57:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 13:57:24 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +
.../events/EntityDeletedHandler.java | 58 ++++++++++-------
.../events/EntityVersionCreatedHandler.java | 58 +++++++++++------
.../events/EntityVersionDeletedHandler.java | 54 +++++++++-------
.../corepersistence/StaleIndexCleanupTest.java | 31 +++++++--
.../PerformanceEntityRebuildIndexTest.java | 2 +
.../collection/guice/CollectionModule.java | 22 ++-----
.../EntityCollectionManagerFactoryImpl.java | 54 +++++++---------
.../impl/EntityCollectionManagerImpl.java | 40 +++++-------
.../collection/impl/EntityDeletedTask.java | 21 +++---
.../impl/EntityVersionCleanupTask.java | 41 ++++++------
.../impl/EntityVersionCleanupTaskTest.java | 68 +++++++++++---------
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 13 ----
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
15 files changed, 252 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index d808ae5..e2b7bd1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -620,6 +620,8 @@ public class CpEntityManager implements EntityManager {
WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause();
handleWriteUniqueVerifyException( entity, wuve );
}
+
+ throw hre;
}
// update in all containing collections and connection indexes
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 33cc988..94ef5e4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -17,52 +17,66 @@
*/
package org.apache.usergrid.corepersistence.events;
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.index.EntityIndex;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
/**
* Delete all Query Index indexes associated with an Entity that has just been deleted.
*/
+@Singleton
public class EntityDeletedHandler implements EntityDeleted {
- private static final Logger logger = LoggerFactory.getLogger(EntityDeletedHandler.class );
+ private static final Logger logger = LoggerFactory.getLogger( EntityDeletedHandler.class );
+
+
+ private final EntityManagerFactory emf;
+
@Inject
- EntityManagerFactory emf;
+ public EntityDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
@Override
- public void deleted(CollectionScope scope, Id entityId, UUID version) {
+ public void deleted( CollectionScope scope, Id entityId, UUID version ) {
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
- if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+ if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" ) ) {
return;
}
- logger.debug("Handling deleted event for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
+ logger.debug( "Handling deleted event for entity {}:{} v {} " + "scope\n name: {}\n owner: {}\n app: {}",
new Object[] {
- entityId.getType(),
- entityId.getUuid(),
- version,
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
+ entityId.getType(), entityId.getUuid(), version, scope.getName(), scope.getOwner(),
+ scope.getApplication()
+ } );
+
+ CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf;
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
+
+ final IndexScope indexScope =
+ new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ),
+ scope.getName() );
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
-// ei.deleteAllVersionsOfEntity( entityId );
+ ei.createBatch().deindex( indexScope, entityId, version ).execute();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 590df61..6270501 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.corepersistence.events;
import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,6 +27,7 @@ import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -35,34 +38,51 @@ import org.apache.usergrid.persistence.model.entity.Entity;
* updated by the Collections module and we react by calling the Query Index module and removing
* any indexes that exist for previous versions of the the Entity.
*/
+@Singleton
public class EntityVersionCreatedHandler implements EntityVersionCreated {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
+
+ private final EntityManagerFactory emf;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
@Inject
- EntityManagerFactory emf;
+ public EntityVersionCreatedHandler( final EntityManagerFactory emf,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.emf = emf;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
@Override
public void versionCreated( final CollectionScope scope, final Entity entity ) {
+ //not op, we're not migrating properly to this. Make this an event
- // This check is for testing purposes and for a test that to be able to dynamically turn
- // off and on delete previous versions so that it can test clean-up on read.
- if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
- return;
- }
-
- logger.debug("Handling versionCreated for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] {
- entity.getId().getType(),
- entity.getId().getUuid(),
- entity.getVersion(),
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
-
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+// // This check is for testing purposes and for a test that to be able to dynamically turn
+// // off and on delete previous versions so that it can test clean-up on read.
+// if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+// return;
+// }
+//
+// logger.debug("Handling versionCreated for entity {}:{} v {} "
+// + "scope\n name: {}\n owner: {}\n app: {}",
+// new Object[] {
+// entity.getId().getType(),
+// entity.getId().getUuid(),
+// entity.getVersion(),
+// scope.getName(),
+// scope.getOwner(),
+// scope.getApplication()});
+//
+// CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
+// final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+//
+//
+//
+//
+//
+//
// ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 127ae46..01848cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.corepersistence.events;
import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
import java.util.List;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
@@ -34,6 +36,10 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Action2;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -43,14 +49,17 @@ import rx.schedulers.Schedulers;
* TODO: do we need this? Don't our version-created and entity-deleted handlers take care of this?
* If we do need it then it should be wired in via GuiceModule in the corepersistence package.
*/
+@Singleton
public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionDeletedHandler.class );
- @Inject
- private SerializationFig serializationFig;
+
+
+
+ private final EntityManagerFactory emf;
@Inject
- private EntityManagerFactory emf;
+ public EntityVersionDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
@Override
@@ -63,40 +72,35 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
return;
}
- logger.debug("Handling versionDeleted count={} event for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] {
- entityVersions.size(),
- entityId.getType(),
- entityId.getUuid(),
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Handling versionDeleted count={} event for entity {}:{} v {} " + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] {
+ entityVersions.size(), entityId.getType(), entityId.getUuid(), scope.getName(), scope.getOwner(),
+ scope.getApplication()
+ } );
+ }
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- final EntityIndexBatch eibatch = ei.createBatch();
-
final IndexScope indexScope = new IndexScopeImpl(
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
scope.getName()
);
- rx.Observable.from(entityVersions)
- .subscribeOn(Schedulers.io())
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ Observable.from( entityVersions )
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
@Override
- public List<MvccEntity> call(List<MvccEntity> entityList) {
- for (MvccEntity entity : entityList) {
- eibatch.deindex(indexScope, entityId, entity.getVersion());
- }
- eibatch.execute();
- return entityList;
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
+ entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
}
- }).toBlocking().last();
+ } ).doOnNext( new Action1<EntityIndexBatch>() {
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch ) {
+ entityIndexBatch.execute();
+ }
+ } ).toBlocking().last();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 8d31b69..ac04dcc 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -246,7 +246,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
*/
- @Test(timeout=30000)
+// @Test(timeout=30000)
+ @Test
public void testCleanupOnDelete() throws Exception {
logger.info("Started testStaleIndexCleanup()");
@@ -316,11 +317,19 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// wait for indexes to be cleared for the deleted entities
count = 0;
+
+
+ //we can't use our candidate result sets here. The repair won't happen since we now have orphaned documents in our index
+ //us the EM so the repair process happens
+
+ Results results = null;
do {
- Thread.sleep(100);
+ //trigger the repair
+ results = queryCollectionEm("things", "select *");
crs = queryCollectionCp("things", "thing", "select *");
- em.refreshIndex();
- } while ( crs.size() > 0 && count++ < 15 );
+ Thread.sleep(100);
+
+ } while ((results.hasCursor() || crs.size() > 0) && count++ < 2000 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
@@ -436,4 +445,18 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
return ei.search( is, SearchTypes.fromTypes( type ), rcq );
}
+
+ /**
+ * Go around EntityManager and execute query directly against Core Persistence.
+ * Results may include stale index entries.
+ */
+ private Results queryCollectionEm( final String collName, final String query ) throws Exception {
+
+ EntityManager em = app.getEntityManager();
+
+
+ final Results results = em.searchCollection( em.getApplicationRef(), collName, Query.fromQL( query ).withLimit( 10000 ) );
+
+ return results;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index def9ed5..f1f165d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -362,6 +362,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
registry.remove( meterName );
logger.info("Rebuilt index");
+ setup.getEmf().refreshIndex();
+
} catch (Exception ex) {
logger.error("Error rebuilding index", ex);
fail();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index 7cd383a..a73d7a7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -18,30 +18,24 @@
package org.apache.usergrid.persistence.collection.guice;
-
import org.safehaus.guicyfig.GuicyFigModule;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerFactoryImpl;
-import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerImpl;
-import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerSyncImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.SerializationModule;
+import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.service.impl.ServiceModule;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -52,8 +46,6 @@ import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.model.entity.Entity;
/**
@@ -72,9 +64,7 @@ public class CollectionModule extends AbstractModule {
install( new SerializationModule() );
install( new ServiceModule() );
- install ( new FactoryModuleBuilder().build( EntityVersionCleanupFactory.class ));
- install ( new FactoryModuleBuilder().build( EntityDeletedFactory.class));
- install ( new FactoryModuleBuilder().build( EntityVersionCreatedFactory.class ));
+ install ( new FactoryModuleBuilder().build( EntityVersionTaskFactory.class ));
// users of this module can add their own implemementations
// for more information: https://github.com/google/guice/wiki/Multibindings
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index bb1b56a..23e375d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -19,23 +19,6 @@
package org.apache.usergrid.persistence.collection.impl;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.netflix.astyanax.Keyspace;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import java.util.concurrent.ExecutionException;
@@ -43,17 +26,32 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+import org.apache.usergrid.persistence.collection.guice.Write;
+import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
@@ -76,9 +74,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final Keyspace keyspace;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
@@ -92,8 +88,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final EntityCollectionManager target = new EntityCollectionManagerImpl( writeStart, writeUpdate, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace, entityVersionTaskFactory,
+ taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -115,9 +111,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
final EntityCacheFig entityCacheFig,
MetricsFactory metricsFactory) {
@@ -134,9 +128,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
this.metricsFactory = metricsFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 50b581a..92b07f0 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,9 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +31,7 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@ -47,11 +45,13 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -59,6 +59,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -68,12 +70,6 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Notification;
import rx.Observable;
@@ -112,9 +108,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
private final Keyspace keyspace;
@@ -144,9 +138,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
@Assisted final CollectionScope collectionScope,
final MetricsFactory metricsFactory
@@ -170,9 +162,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.collectionScope = collectionScope;
@@ -216,8 +206,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public void call(final Entity entity) {
//TODO fire the created task first then the entityVersioncleanup
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
- taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
+ taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
+ taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entityId,
+ entity.getVersion(), false ));
//post-processing to come later. leave it empty for now.
}
}).doOnError(rollback)
@@ -252,8 +243,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityDeletedFactory
- .getTask(collectionScope, entity.getId(), entity.getVersion());
+ Task<Void> task = entityVersionTaskFactory
+ .getDeleteTask( collectionScope, entity.getId(), entity.getVersion() );
taskExecutor.submit(task);
return entity.getId();
}
@@ -400,8 +391,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
logger.debug("sending entity to the queue");
//we an update, signal the fix
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope, entity));
+ taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
+ taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entity.getId(), entity.getVersion(), false) );
//TODO T.N Change this to fire a task
// Observable.from( new CollectionIoEvent<Id>(collectionScope,
// entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index 9ff4f56..83f165d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@ -22,7 +22,6 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
@@ -46,7 +45,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
public class EntityDeletedTask implements Task<Void> {
private static final Logger LOG = LoggerFactory.getLogger(EntityDeletedTask.class);
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final Set<EntityDeleted> listeners;
@@ -56,16 +55,16 @@ public class EntityDeletedTask implements Task<Void> {
@Inject
- public EntityDeletedTask(
- EntityVersionCleanupFactory entityVersionCleanupFactory,
+ public EntityDeletedTask(
+ EntityVersionTaskFactory entityVersionTaskFactory,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
@ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
final Set<EntityDeleted> listeners, // MUST be a set or Guice will not inject
- @Assisted final CollectionScope collectionScope,
- @Assisted final Id entityId,
+ @Assisted final CollectionScope collectionScope,
+ @Assisted final Id entityId,
@Assisted final UUID version) {
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.logEntrySerializationStrategy = logEntrySerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
this.listeners = listeners;
@@ -81,7 +80,7 @@ public class EntityDeletedTask implements Task<Void> {
new Object[] { collectionScope, entityId, version }, throwable );
}
-
+
@Override
public Void rejected() {
try {
@@ -94,11 +93,11 @@ public class EntityDeletedTask implements Task<Void> {
return null;
}
-
+
@Override
- public Void call() throws Exception {
+ public Void call() throws Exception {
- entityVersionCleanupFactory.getTask( collectionScope, entityId, version ).call();
+ entityVersionTaskFactory.getCleanupTask( collectionScope, entityId, version, true ).call();
fireEvents();
final MutationBatch entityDelete = entitySerializationStrategy.delete(collectionScope, entityId, version);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index efecdeb..03a5ff6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -55,7 +55,7 @@ import rx.schedulers.Schedulers;
/**
- * Cleans up previous versions from the specified version. Note that this means the version
+ * Cleans up previous versions from the specified version. Note that this means the version
* passed in the io event is retained, the range is exclusive.
*/
public class EntityVersionCleanupTask implements Task<Void> {
@@ -74,10 +74,11 @@ public class EntityVersionCleanupTask implements Task<Void> {
private final CollectionScope scope;
private final Id entityId;
private final UUID version;
+ private final int numToSkip;
@Inject
- public EntityVersionCleanupTask(
+ public EntityVersionCleanupTask(
final SerializationFig serializationFig,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
@ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
@@ -86,7 +87,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
@Assisted final CollectionScope scope,
@Assisted final Id entityId,
- @Assisted final UUID version ) {
+ @Assisted final UUID version,
+ @Assisted final boolean includeVersion) {
this.serializationFig = serializationFig;
this.logEntrySerializationStrategy = logEntrySerializationStrategy;
@@ -97,6 +99,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
this.scope = scope;
this.entityId = entityId;
this.version = version;
+
+ numToSkip = includeVersion? 0: 1;
}
@@ -136,7 +140,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
}
})
//buffer them for efficiency
- .skip(1)
+ .skip(numToSkip)
.buffer(serializationFig.getBufferSize()).doOnNext(
new Action1<List<MvccEntity>>() {
@Override
@@ -146,27 +150,28 @@ public class EntityVersionCleanupTask implements Task<Void> {
final MutationBatch logBatch = keyspace.prepareMutationBatch();
for (MvccEntity mvccEntity : mvccEntities) {
- if (!mvccEntity.getEntity().isPresent()) {
- continue;
- }
-
final UUID entityVersion = mvccEntity.getVersion();
- final Entity entity = mvccEntity.getEntity().get();
- //remove all unique fields from the index
- for (final Field field : entity.getFields()) {
- if (!field.isUnique()) {
- continue;
+
+ //if the entity is present process the fields
+ if(mvccEntity.getEntity().isPresent()) {
+ final Entity entity = mvccEntity.getEntity().get();
+
+ //remove all unique fields from the index
+ for ( final Field field : entity.getFields() ) {
+ if ( !field.isUnique() ) {
+ continue;
+ }
+ final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
+ final MutationBatch deleteMutation =
+ uniqueValueSerializationStrategy.delete( scope, unique );
+ batch.mergeShallow( deleteMutation );
}
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion);
- final MutationBatch deleteMutation =
- uniqueValueSerializationStrategy.delete(scope,unique);
- batch.mergeShallow(deleteMutation);
}
final MutationBatch entityDelete = entitySerializationStrategy
.delete(scope, entityId, mvccEntity.getVersion());
- entityBatch.mergeShallow(entityDelete);
+ entityBatch.mergeShallow( entityDelete );
final MutationBatch logDelete = logEntrySerializationStrategy
.delete(scope, entityId, version);
logBatch.mergeShallow(logDelete);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index d0a87c3..f2fb95b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -79,7 +79,7 @@ public class EntityVersionCleanupTaskTest {
@Test(timeout=10000)
- public void noListenerOneVersion()
+ public void noListenerOneVersion()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -109,7 +109,7 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
@@ -133,7 +133,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -148,10 +149,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory(
@@ -175,7 +176,7 @@ public class EntityVersionCleanupTaskTest {
* Tests the cleanup task on the first version created
*/
@Test(timeout=10000)
- public void noListenerNoVersions()
+ public void noListenerNoVersions()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -208,14 +209,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, 1 );
@@ -233,7 +234,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -249,10 +251,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -267,7 +269,7 @@ public class EntityVersionCleanupTaskTest {
// These last two verify statements do not make sense. We cannot assert that the entity
- // and log batches are never called. Even if there are no listeners the entity delete
+ // and log batches are never called. Even if there are no listeners the entity delete
// cleanup task will still run to do the normal cleanup.
//
// verify( entityBatch, never() ).execute();
@@ -276,7 +278,7 @@ public class EntityVersionCleanupTaskTest {
@Test(timeout=10000)
- public void singleListenerSingleVersion()
+ public void singleListenerSingleVersion()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -317,14 +319,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
@@ -343,7 +345,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -361,10 +364,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -421,7 +424,7 @@ public class EntityVersionCleanupTaskTest {
final int sizeToReturn = 10;
- final CountDownLatch latch = new CountDownLatch(
+ final CountDownLatch latch = new CountDownLatch(
sizeToReturn/serializationFig.getBufferSize() * 3 );
final EntityVersionDeletedTest listener1 = new EntityVersionDeletedTest( latch );
@@ -436,13 +439,13 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
// mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
@@ -456,7 +459,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -474,10 +478,10 @@ public class EntityVersionCleanupTaskTest {
Entity entity = new Entity( entityId );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.of(entity)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.of(entity)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -543,7 +547,7 @@ public class EntityVersionCleanupTaskTest {
final int listenerCount = 5;
- final CountDownLatch latch = new CountDownLatch(
+ final CountDownLatch latch = new CountDownLatch(
sizeToReturn/serializationFig.getBufferSize() * listenerCount );
final Semaphore waitSemaphore = new Semaphore( 0 );
@@ -565,14 +569,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
@@ -591,7 +595,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -714,7 +719,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -768,7 +774,7 @@ public class EntityVersionCleanupTaskTest {
@Override
- public void versionDeleted( final CollectionScope scope, final Id entityId,
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
final List<MvccEntity> entityVersion ) {
invocationLatch.countDown();
}
@@ -786,7 +792,7 @@ public class EntityVersionCleanupTaskTest {
@Override
- public void versionDeleted( final CollectionScope scope, final Id entityId,
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
final List<MvccEntity> entityVersion ) {
//wait for unblock to happen before counting down invocation latches
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 16b3ff9..92312d2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -101,6 +101,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
public EntityIndexBatch index( final IndexScope indexScope, final Entity entity ) {
IndexValidationUtils.validateIndexScope( indexScope );
ValidationUtils.verifyEntityWrite( entity );
+ ValidationUtils.verifyVersion( entity.getVersion() );
final String context = createContextName(indexScope);
@@ -161,7 +162,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
String[] indexes = entityIndex.getIndexes(AliasedEntityIndex.AliasType.Read);
//get the default index if no alias exists yet
if(indexes == null ||indexes.length == 0){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 3a11ec7..16f4b3f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -113,9 +113,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer addWriteAliasTimer;
private final Timer addReadAliasTimer;
private final Timer searchTimer;
- private final Timer allVersionsTimerFuture;
- private final Timer deletePreviousTimerFuture;
- private final Meter errorMeter;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -190,16 +187,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
.getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
- this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
- this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
- this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
- this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
-
- this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 5259a26..d55073a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -273,7 +273,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
public void call( final BulkRequestBuilder bulkRequestBuilder ) {
sendRequest( bulkRequestBuilder );
}
- } ).toBlocking().last();
+ } ).toBlocking().lastOrDefault(null);
//call back all futures
Observable.from(operationMessages)
[36/50] incubator-usergrid git commit: [USERGRID-396] Added a timer
to track how long repairing the entities takes. Fixed params in entity
manager. Removed getEntityByAlias as it was just a filler method and is no
longer used. Removed some useless commen
Posted by to...@apache.org.
[USERGRID-396] Added a timer to track how long repairing the entities takes.
Fixed params in entity manager.
Removed getEntityByAlias as it was just a filler method and is no longer used.
Removed some useless comments in abstract collection service.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/24a48a75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/24a48a75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/24a48a75
Branch: refs/heads/USERGRID-405
Commit: 24a48a756dbd56094a6c75cab8befc169b730776
Parents: c45a3ed
Author: GERey <gr...@apigee.com>
Authored: Fri Mar 13 11:07:06 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Mar 13 11:07:06 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityManager.java | 17 +++++------------
.../apache/usergrid/persistence/EntityManager.java | 15 ++++-----------
.../persistence/cassandra/EntityManagerImpl.java | 7 -------
.../services/AbstractCollectionService.java | 4 ----
4 files changed, 9 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24a48a75/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 0f4fdc9..a108f25 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -207,6 +207,7 @@ public class CpEntityManager implements EntityManager {
private Timer entGetEntityCountersTimer;
private Timer esIndexEntityCollectionTimer;
private Timer entRevokeRolePermissionsTimer;
+ private Timer entGetRepairedEntityTimer;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -267,6 +268,8 @@ public class CpEntityManager implements EntityManager {
.getTimer( CpEntityManager.class, "cp.entity.es.index.entity.to.collection.timer" );
this.entRevokeRolePermissionsTimer =
this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+ this.entGetRepairedEntityTimer = this.metricsFactory
+ .getTimer( CpEntityManager.class, "get.repaired.entity.timer" );
// set to false for now
this.skipAggregateCounters = false;
@@ -814,6 +817,7 @@ public class CpEntityManager implements EntityManager {
String propertyName = Schema.getDefaultSchema().aliasProperty( collName );
+ Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
//TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
@@ -824,6 +828,7 @@ public class CpEntityManager implements EntityManager {
FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+ repairedEntityGet.stop();
if(fieldSet.isEmpty()) {
return null;
}
@@ -832,18 +837,6 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Entity getEntityByAlias(String collectionType, String aliasType) throws Exception {
- EntityRef newEntityRef = getAlias(collectionType,aliasType);
-
- if(newEntityRef == null) {
- return null;
- }
-
- return get( newEntityRef );
- }
-
-
- @Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24a48a75/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index eb6379b..5657d4f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -17,9 +17,6 @@
package org.apache.usergrid.persistence;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.index.query.Query;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -28,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import me.prettyprint.hector.api.mutation.Mutator;
-import rx.Observable;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
@@ -38,7 +34,6 @@ import org.apache.usergrid.persistence.entities.Role;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query.Level;
-import org.apache.usergrid.persistence.model.field.Field;
/**
@@ -99,14 +94,12 @@ public interface EntityManager {
public void createApplicationCollection( String entityType ) throws Exception;
- Entity getEntityByAlias( String collectionType, String aliasType ) throws Exception;
-
public EntityRef getAlias( String aliasType, String alias ) throws Exception;
/**
* Get the entity ref from the value
*
- * @param ownerId The owner Id of the collection
+ * @param ownerRef The owner Id of the collection
* @param collectionName The name of the collection
* @param aliasValue The value of the alias
*/
@@ -118,7 +111,7 @@ public interface EntityManager {
/**
* Get aliases from the index with the given value
*
- * @param ownerId The id of the collection owner
+ * @param ownerRef The id of the collection owner
* @param collectionName The name of the collection
* @param aliases The alias property
*/
@@ -454,7 +447,7 @@ public interface EntityManager {
* Gets the entities of the specified type connected to the specified entity, optionally
* matching the specified connection types and/or entity types. Returns a list of entity ids.
*
- * @param entityId an entity reference
+ * @param entityRef an entity reference
* @param connectionType type of connection or null.
* @param connectedEntityType type of entity or null.
*
@@ -471,7 +464,7 @@ public interface EntityManager {
* <p/>
* e.g. "get users who have favorited this place"
*
- * @param entityId an entity reference
+ * @param entityRef an entity reference
* @param connectionType type of connection or null.
* @param connectedEntityType type of entity or null.
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24a48a75/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index a019e86..3993285 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -1771,13 +1771,6 @@ public class EntityManagerImpl implements EntityManager {
batchExecute( m, CassandraService.RETRY_COUNT );
}
-
- @Override
- public Entity getEntityByAlias( final String collectionType, final String aliasType ) throws Exception {
- throw new UnsupportedOperationException("Not supported.");
- }
-
-
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
return getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24a48a75/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 0820737..e404321 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -46,13 +46,9 @@ public class AbstractCollectionService extends AbstractService {
public AbstractCollectionService() {
- // addSet("indexes");
declareMetadataType( "indexes" );
}
- // cname/id/
-
-
@Override
public Entity getEntity( ServiceRequest request, UUID uuid ) throws Exception {
if ( !isRootService() ) {
[48/50] incubator-usergrid git commit: Second pass at fixing the merge
Posted by to...@apache.org.
Second pass at fixing the merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/762e43e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/762e43e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/762e43e1
Branch: refs/heads/USERGRID-405
Commit: 762e43e1ee9dfda1a07b9ba6ea7cd13cede06afc
Parents: 7b5d222
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 17:26:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 17:26:25 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 37 +++++------
.../corepersistence/CpEntityManagerFactory.java | 21 +++----
.../events/EntityVersionDeletedHandler.java | 8 +--
.../collection/EntityCollectionManager.java | 6 --
.../EntityCollectionManagerFactoryImpl.java | 22 ++++---
.../impl/EntityCollectionManagerImpl.java | 65 ++++++++------------
.../impl/EntityVersionCleanupTask.java | 9 +--
.../MvccEntitySerializationStrategyV1Impl.java | 2 +-
.../MvccEntitySerializationStrategyV3Impl.java | 1 -
.../migration/MvccEntityDataMigrationImpl.java | 8 +--
.../collection/EntityCollectionManagerIT.java | 2 +-
.../impl/EntityVersionCleanupTaskTest.java | 13 ++--
...ntitySerializationStrategyProxyV1_3Test.java | 3 +
...ntitySerializationStrategyProxyV2_3Test.java | 3 +
...ccEntitySerializationStrategyV1ImplTest.java | 2 +
.../impl/GraphManagerFactoryImpl.java | 17 ++---
16 files changed, 104 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 5da7067..2e9b780 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -15,30 +15,37 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.inject.AbstractModule;
-import com.google.inject.Provider;
-import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.corepersistence.migration.EntityDataMigration;
-import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
-import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+import org.springframework.context.ApplicationContext;
+
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.corepersistence.migration.CoreMigration;
+import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
+import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
+import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
-import org.apache.usergrid.persistence.map.guice.MapModule;
-import org.apache.usergrid.persistence.queue.guice.QueueModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provider;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
/**
@@ -74,7 +81,7 @@ public class CoreModule extends AbstractModule {
@Override
public void configureMigrationProvider() {
- bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).to(
+ bind(new TypeLiteral<MigrationDataProvider<EntityIdScope>>(){}).to(
AllEntitiesInSystemImpl.class );
}
} );
@@ -95,12 +102,6 @@ public class CoreModule extends AbstractModule {
bind(ManagerCache.class).to( CpManagerCache.class );
- Multibinder<DataMigration> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), DataMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
- dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
-
Multibinder<EntityDeleted> entityBinder =
Multibinder.newSetBinder(binder(), EntityDeleted.class);
entityBinder.addBinding().to(EntityDeletedHandler.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 01dfa1f..f76b9fc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,15 +15,14 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.slf4j.Logger;
@@ -48,9 +47,8 @@ import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -59,7 +57,6 @@ import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.persistence.exceptions.OrganizationAlreadyExistsException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -71,13 +68,13 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
+import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
-import com.yammer.metrics.annotation.Metered;
import rx.Observable;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 95ff9dc..c45949b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -85,7 +85,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
final IndexScope indexScope = new IndexScopeImpl(
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
@@ -93,10 +93,10 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
);
Observable.from( entityVersions )
- .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccLogEntry>() {
@Override
- public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
- entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccLogEntry mvccLogEntry ) {
+ entityIndexBatch.deindex( indexScope, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion() );
}
} ).doOnNext( new Action1<EntityIndexBatch>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 35fc5d4..6532ee6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -75,12 +75,6 @@ public interface EntityCollectionManager {
*/
public Observable<EntitySet> load(Collection<Id> entityIds);
- /**
- * Takes the change and reloads an entity with all changes applied in this entity applied.
- * The resulting entity from calling load will be the previous version of this entity plus
- * the entity in this object applied to it.
- */
- public Observable<Entity> update ( Entity entity );
/**
* Returns health of entity data store.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index ac82181..409467c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -26,18 +26,23 @@ import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@ -45,6 +50,7 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.Keyspace;
@@ -72,25 +78,21 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
- private EntityCacheFig entityCacheFig;
- private SerializationFig serializationFig;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
.build( new CacheLoader<CollectionScope, EntityCollectionManager>() {
public EntityCollectionManager load( CollectionScope scope ) {
-
//create the target EM that will perform logic
final EntityCollectionManager target = new EntityCollectionManagerImpl(
writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
return proxy;
-// return target;
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 001e0db..f565fab 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,28 +19,22 @@
package org.apache.usergrid.persistence.collection.impl;
-import java.util.*;
-
-import com.netflix.astyanax.MutationBatch;
-import org.apache.usergrid.persistence.collection.*;
-import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@ -52,13 +46,11 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimist
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -75,17 +67,12 @@ import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.OperationResult;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Notification;
import rx.Observable;
@@ -224,19 +211,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
entity.getVersion(), false ));
//post-processing to come later. leave it empty for now.
}
- }).doOnError(rollback)
- .doOnEach(new Action1<Notification<? super Entity>>() {
+ }).doOnError( rollback )
+ .doOnEach( new Action1<Notification<? super Entity>>() {
@Override
- public void call(Notification<? super Entity> notification) {
+ public void call( Notification<? super Entity> notification ) {
writeMeter.mark();
}
- })
- .doOnCompleted(new Action0() {
+ } )
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -250,7 +237,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final Timer.Context timer = deleteTimer.time();
Observable<Id> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId))
.map(markStart)
- .doOnNext(markCommit)
+ .doOnNext( markCommit )
.map(new Func1<CollectionIoEvent<MvccEntity>, Id>() {
@Override
@@ -269,12 +256,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
deleteMeter.mark();
}
})
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
return o;
}
@@ -300,18 +287,18 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return Observable.from(entity.getEntity().get());
}
})
- .doOnNext(new Action1<Entity>() {
+ .doOnNext( new Action1<Entity>() {
@Override
- public void call(Entity entity) {
+ public void call( Entity entity ) {
loadMeter.mark();
}
- })
- .doOnCompleted(new Action0() {
+ } )
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -347,12 +334,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
loadMeter.mark();
}
})
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
@@ -505,12 +492,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}
} )
- .doOnCompleted(new Action0() {
+ .doOnCompleted( new Action0() {
@Override
public void call() {
timer.stop();
}
- });
+ } );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 27c3db8..55d135b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -26,10 +26,12 @@ import java.util.UUID;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
@@ -86,7 +88,6 @@ public class EntityVersionCleanupTask implements Task<Void> {
public EntityVersionCleanupTask(
final SerializationFig serializationFig,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final Keyspace keyspace,
final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
@@ -104,7 +105,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
this.entityId = entityId;
this.version = version;
- includeVersion = includeVersion;
+ this.includeVersion = includeVersion;
}
@@ -146,7 +147,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
.skipWhile( new Func1<UniqueValue, Boolean>() {
@Override
public Boolean call( final UniqueValue uniqueValue ) {
- return version.equals( uniqueValue.getEntityVersion() );
+ return !includeVersion && version.equals( uniqueValue.getEntityVersion() );
}
} )
//buffer our buffer size, then roll them all up in a single batch mutation
@@ -185,7 +186,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
.skipWhile( new Func1<MvccLogEntry, Boolean>() {
@Override
public Boolean call( final MvccLogEntry mvccLogEntry ) {
- return version.equals( mvccLogEntry.getVersion() );
+ return !includeVersion && version.equals( mvccLogEntry.getVersion() );
}
} )
//buffer them for efficiency
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index 6f8525c..40054ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -194,7 +194,7 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
// it's been deleted, remove it
if ( Arrays.equals( STATE_DELETED, state ) ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ return new EntityWrapper( MvccEntity.Status.DELETED, Optional.<Entity>absent() );
}
Entity storedEntity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 6ad18bb..ace076b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -310,7 +310,6 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
Preconditions.checkNotNull( version, "version is required" );
- TimeUUIDUtils.getMicrosTimeFromUUID(version);
return doWrite( collectionScope, entityId, version, new RowOp() {
@Override
public void doOp( final ColumnListMutation<Boolean> colMutation ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
index beaaef9..bd4eafc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/MvccEntityDataMigrationImpl.java
@@ -30,9 +30,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
+import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
@@ -75,7 +75,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
private final Keyspace keyspace;
private final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final EntityVersionTaskFactory entityVersionCleanupFactory;
private final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3;
@@ -84,7 +84,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
public MvccEntityDataMigrationImpl( final Keyspace keyspace,
final VersionedMigrationSet<MvccEntitySerializationStrategy> allVersions,
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
+ final EntityVersionTaskFactory entityVersionCleanupFactory,
final MvccEntitySerializationStrategyV3Impl mvccEntitySerializationStrategyV3
) {
@@ -227,7 +227,7 @@ public class MvccEntityDataMigrationImpl implements DataMigration<EntityIdScope>
totalBatch.mergeShallow( mb );
}
- final EntityVersionCleanupTask task = entityVersionCleanupFactory.getTask( message.scope, message.entity.getId(), version );
+ final EntityVersionCleanupTask task = entityVersionCleanupFactory.getCleanupTask( message.scope, message.entity.getId(), version, false );
entityVersionCleanupTasks.add( task );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index c92bb74..f6d9782 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index b4921a3..787ab30 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -128,7 +128,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -209,8 +209,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
- new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId, version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -290,7 +289,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -390,7 +389,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -517,7 +516,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false);
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -645,7 +644,7 @@ public class EntityVersionCleanupTaskTest {
EntityVersionCleanupTask cleanupTask =
new EntityVersionCleanupTask( serializationFig, less, uvss, keyspace, listeners, appScope, entityId,
- version );
+ version, false );
final MutationBatch newBatch = mock( MutationBatch.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
index 42b10c4..56abfd8 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1_3Test.java
@@ -34,9 +34,12 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class MvccEntitySerializationStrategyProxyV1_3Test extends MvccEntitySerializationStrategyImplTest {
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
index da496f7..b372fbf 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2_3Test.java
@@ -34,9 +34,12 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class MvccEntitySerializationStrategyProxyV2_3Test extends MvccEntitySerializationStrategyV2Test {
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index 4e0c8cc..b0dba5d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -49,6 +49,8 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import net.jcip.annotations.NotThreadSafe;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/762e43e1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
index 732f0bf..7c88c33 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/GraphManagerFactoryImpl.java
@@ -26,6 +26,8 @@ import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.Assisted;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.GraphManager;
@@ -51,29 +53,28 @@ public class GraphManagerFactoryImpl implements GraphManagerFactory {
private final GraphFig graphFig;
private final EdgeDeleteListener edgeDeleteListener;
private final NodeDeleteListener nodeDeleteListener;
+ private final MetricsFactory metricsFactory;
private LoadingCache<ApplicationScope, GraphManager> gmCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, GraphManager>() {
public GraphManager load(
ApplicationScope scope ) {
- return new GraphManagerImpl(edgeMetadataSerialization,edgeSerialization,nodeSerialization,graphFig,edgeDeleteListener,nodeDeleteListener,scope);
+ return new GraphManagerImpl(edgeMetadataSerialization,edgeSerialization,nodeSerialization,graphFig,edgeDeleteListener,nodeDeleteListener,scope, metricsFactory);
}
} );
@Inject
- public GraphManagerFactoryImpl(final EdgeMetadataSerialization edgeMetadataSerialization,
- final EdgeSerialization edgeSerialization,
- final NodeSerialization nodeSerialization,
- final GraphFig graphFig,
- final EdgeDeleteListener edgeDeleteListener,
- final NodeDeleteListener nodeDeleteListener
- ){
+ public GraphManagerFactoryImpl( final EdgeMetadataSerialization edgeMetadataSerialization, final
+ EdgeSerialization edgeSerialization,
+ final NodeSerialization nodeSerialization, final GraphFig graphFig, final EdgeDeleteListener edgeDeleteListener,
+ final NodeDeleteListener nodeDeleteListener, final MetricsFactory metricsFactory ){
this.edgeMetadataSerialization = edgeMetadataSerialization;
this.edgeSerialization = edgeSerialization;
this.nodeSerialization = nodeSerialization;
this.graphFig = graphFig;
this.edgeDeleteListener = edgeDeleteListener;
this.nodeDeleteListener = nodeDeleteListener;
+ this.metricsFactory = metricsFactory;
}
@Override
[50/50] incubator-usergrid git commit: Removed Hystrix. Causing
thread OOM issues and not really used in production.
Posted by to...@apache.org.
Removed Hystrix. Causing thread OOM issues and not really used in production.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e6ed4a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e6ed4a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e6ed4a1
Branch: refs/heads/USERGRID-405
Commit: 5e6ed4a12c7aa154f1411d85c6fcce56fbcd4367
Parents: b09dc43
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 18:23:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 18:23:58 2015 -0600
----------------------------------------------------------------------
.../core/astyanax/MultiRowColumnIterator.java | 12 ++-
.../core/hystrix/HystrixCassandra.java | 94 --------------------
.../persistence/core/task/TaskExecutor.java | 4 +-
.../graph/impl/GraphManagerImpl.java | 30 +++++--
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 16 +++-
.../impl/stage/NodeDeleteListenerImpl.java | 16 +++-
.../impl/NodeSerializationImpl.java | 36 ++++----
.../shard/count/NodeShardApproximationImpl.java | 4 +-
.../NodeShardCounterSerializationImpl.java | 25 +++---
.../shard/impl/NodeShardAllocationImpl.java | 16 +++-
.../shard/impl/ShardGroupCompactionImpl.java | 64 ++++---------
12 files changed, 128 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
index fdc4768..667992c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
@@ -33,10 +33,8 @@ import java.util.NoSuchElementException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
-
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.ColumnList;
@@ -184,7 +182,13 @@ public class MultiRowColumnIterator<R, C, T> implements Iterator<T> {
keyspace.prepareQuery( cf ).setConsistencyLevel( consistencyLevel ).getKeySlice( rowKeys )
.withColumnRange( rangeBuilder.build() );
- final Rows<R, C> result = HystrixCassandra.user( query ).getResult();
+ final Rows<R, C> result;
+ try {
+ result = query.execute().getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
//now aggregate them together
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
deleted file mode 100644
index ab71782..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hystrix/HystrixCassandra.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.hystrix;
-
-
-import com.netflix.astyanax.Execution;
-import com.netflix.astyanax.connectionpool.OperationResult;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.hystrix.HystrixCommand;
-import com.netflix.hystrix.HystrixCommandGroupKey;
-import com.netflix.hystrix.HystrixThreadPoolProperties;
-
-
-/**
- * A utility class that creates graph observables wrapped in Hystrix for timeouts and circuit breakers.
- */
-public class HystrixCassandra {
-
-
-
-
- /**
- * Command group used for realtime user commands
- */
- public static final HystrixCommand.Setter
- USER_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "user" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
-
- /**
- * Command group for asynchronous operations
- */
- public static final HystrixCommand.Setter
- ASYNC_GROUP = HystrixCommand.Setter.withGroupKey( HystrixCommandGroupKey.Factory.asKey( "async" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
-
-
- /**
- * Execute an user operation
- */
- public static <R> OperationResult<R> user( final Execution<R> execution) {
- return new HystrixCommand<OperationResult<R>>( USER_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
- try {
- return execution.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( e );
- }
- }
- }.execute();
- }
-
-
- /**
- * Execute an an async operation
- */
- public static <R> OperationResult<R> async( final Execution<R> execution) {
-
-
- return new HystrixCommand<OperationResult<R>>( ASYNC_GROUP ) {
-
- @Override
- protected OperationResult<R> run() {
- try {
- return execution.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( e );
- }
- }
- }.execute();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
index 5e9aa4c..5728d2e 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/TaskExecutor.java
@@ -25,7 +25,7 @@ import com.google.common.util.concurrent.ListenableFuture;
/**
* An interface for execution of tasks
*/
-public interface TaskExecutor {
+public interface TaskExecutor {
/**
* Submit the task asynchronously
@@ -37,5 +37,5 @@ public interface TaskExecutor {
* Stop the task executor without waiting for scheduled threads to run
*/
public void shutdown();
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 246bbca..26d06ad 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -25,14 +25,11 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -54,10 +51,12 @@ import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Notification;
import rx.Observable;
@@ -202,7 +201,12 @@ public class GraphManagerImpl implements GraphManager {
mutation.mergeShallow( edgeMutation );
- HystrixCassandra.user( mutation );
+ try {
+ mutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
return edge;
}
@@ -241,7 +245,12 @@ public class GraphManagerImpl implements GraphManager {
LOG.debug("Marking edge {} as deleted to commit log", edge);
- HystrixCassandra.user(edgeMutation);
+ try {
+ edgeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
//HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
@@ -285,7 +294,12 @@ public class GraphManagerImpl implements GraphManager {
LOG.debug( "Marking node {} as deleted to node mark", node );
- HystrixCassandra.user( nodeMutation );
+ try {
+ nodeMutation.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute mutation", e );
+ }
//HystrixCassandra.async(nodeDeleteListener.receive(scope, id, eventTimestamp )).subscribeOn(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
index 0137ba4..7dca0ce 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
@@ -26,7 +26,6 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -94,7 +94,12 @@ public class EdgeDeleteRepairImpl implements EdgeDeleteRepair {
//remove from storage
- HystrixCassandra.async(storageSerialization.deleteEdge( scope, edge, timestamp ));
+ try {
+ storageSerialization.deleteEdge( scope, edge, timestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
index 0e1c4e2..ab141f7 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
@@ -28,7 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -47,6 +46,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -188,7 +188,12 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
+ "Mutation has {} rows to mutate ",
edgeType, batch.getRowCount() );
- HystrixCassandra.async( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
}
@@ -219,7 +224,12 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
LOG.debug( "Type {} has no subtypes in use as of maxTimestamp {}. Deleting type.", edgeType,
maxTimestamp );
- HystrixCassandra.async( serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ) );
+ try {
+ serialization.removeEdgeType( scope, node, edgeType, maxTimestamp ).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index f167f0c..e8c224e 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -29,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -47,6 +46,7 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action0;
@@ -129,7 +129,12 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
.doOnCompleted( new Action0() {
@Override
public void call() {
- HystrixCassandra.async(nodeSerialization.delete( scope, node, maxVersion.get() ));
+ try {
+ nodeSerialization.delete( scope, node, maxVersion.get()).execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
}
} );
}
@@ -210,7 +215,12 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
targetNodes.add( new TargetPair( edge.getTargetNode(), edge.getType() ) );
}
- HystrixCassandra.async( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
//now delete meta data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
index 2cc0391..18062c4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
@@ -38,7 +38,6 @@ import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -52,6 +51,7 @@ import com.google.common.base.Preconditions;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.Row;
@@ -150,22 +150,23 @@ public class NodeSerializationImpl implements NodeSerialization, Migration {
keyspace.prepareQuery( GRAPH_DELETE ).setConsistencyLevel( fig.getReadCL() );
+
+ Column<Boolean> result = null;
try {
- Column<Boolean> result = HystrixCassandra
- .user( query.getKey( ScopedRowKey.fromKey( scope.getApplication(), node ) ).getColumn( COLUMN_NAME ) )
+ result = query.getKey( ScopedRowKey.fromKey( scope.getApplication(), node ) ).getColumn( COLUMN_NAME ).execute()
.getResult();
-
- return Optional.of( result.getLongValue() );
}
- catch (RuntimeException re ) {
- if(re.getCause().getCause() instanceof NotFoundException) {
- //swallow, there's just no column
- return Optional.absent();
- }
-
- throw re;
+ catch(NotFoundException nfe){
+ //swallow, there's just no column
+ return Optional.absent();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
}
+ return Optional.of( result.getLongValue() );
+
+
}
@@ -193,9 +194,14 @@ public class NodeSerializationImpl implements NodeSerialization, Migration {
}
- final Rows<ScopedRowKey<Id>, Boolean> results = HystrixCassandra
- .user( query.getRowSlice( keys ).withColumnSlice( Collections.singletonList( COLUMN_NAME ) ) )
- .getResult();
+ final Rows<ScopedRowKey<Id>, Boolean> results;
+ try {
+ results = query.getRowSlice( keys ).withColumnSlice( Collections.singletonList( COLUMN_NAME )).execute()
+ .getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
for ( Row<ScopedRowKey<Id>, Boolean> row : results ) {
Column<Boolean> column = row.getColumns().getColumnByName( COLUMN_NAME );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
index 47243e5..a47d528 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
@@ -30,7 +30,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import com.netflix.astyanax.MutationBatch;
import com.netflix.hystrix.HystrixCommand;
+import com.netflix.hystrix.HystrixCommandGroupKey;
import rx.functions.Action0;
import rx.schedulers.Schedulers;
@@ -229,7 +229,7 @@ public class NodeShardApproximationImpl implements NodeShardApproximation {
/**
* Execute the command in hystrix to avoid slamming cassandra
*/
- new HystrixCommand( HystrixCassandra.ASYNC_GROUP ) {
+ new HystrixCommand( HystrixCommandGroupKey.Factory.asKey("BatchCounterRollup") ) {
@Override
protected Void run() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
index 524a0cf..6934275 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
@@ -33,9 +33,8 @@ import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -50,6 +49,7 @@ import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.OperationResult;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.CompositeBuilder;
@@ -117,24 +117,19 @@ public class NodeShardCounterSerializationImpl implements NodeShardCounterSerial
final ScopedRowKey rowKey = ScopedRowKey.fromKey( key.scope.getApplication(), key );
+ OperationResult<Column<Boolean>> column = null;
try {
- OperationResult<Column<Boolean>> column = HystrixCassandra.user(
- keyspace.prepareQuery( EDGE_SHARD_COUNTS ).getKey( rowKey ).getColumn( true ) );
-
- return column.getResult().getLongValue();
+ column = keyspace.prepareQuery( EDGE_SHARD_COUNTS ).getKey( rowKey ).getColumn( true ).execute();
}
//column not found, return 0
- catch ( RuntimeException re ) {
-
- final Throwable cause = re.getCause();
-
- if(cause != null && cause.getCause() instanceof NotFoundException) {
- return 0;
- }
-
- throw re;
+ catch ( NotFoundException nfe ) {
+ return 0;
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to read from cassandra", e );
}
+ return column.getResult().getLongValue();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 0d34b63..ad64338 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -27,7 +27,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.GraphFig;
@@ -49,6 +48,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.util.TimeUUIDUtils;
@@ -110,7 +110,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, MIN_SHARD, directedEdgeMeta );
- HystrixCassandra.user( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
existingShards = Collections.singleton( MIN_SHARD ).iterator();
}
@@ -232,7 +237,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final MutationBatch batch = this.edgeShardSerialization.writeShardMeta( scope, newShard, directedEdgeMeta );
- HystrixCassandra.user( batch );
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e6ed4a1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 9f8efc8..6135121 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -29,13 +29,7 @@ import java.util.Iterator;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.RejectedExecutionHandler;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
@@ -43,7 +37,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -74,6 +67,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
/**
@@ -207,8 +201,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
if ( edgeCount % maxWorkSize == 0 ) {
try {
- HystrixCassandra.async( newRowBatch );
- HystrixCassandra.async( deleteRowBatch );
+ newRowBatch.execute();
+ deleteRowBatch.execute();
}
catch ( Throwable t ) {
LOG.error( "Unable to move edges from shard {} to shard {}", sourceShard, targetShard );
@@ -219,8 +213,8 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
try {
- HystrixCassandra.async( newRowBatch );
- HystrixCassandra.async( deleteRowBatch );
+ newRowBatch.execute();
+ deleteRowBatch.execute();
}
catch ( Throwable t ) {
LOG.error( "Unable to move edges to target shard {}", targetShard );
@@ -260,7 +254,12 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- HystrixCassandra.async( shardRemovalRollup );
+ try {
+ shardRemovalRollup.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
LOG.info( "Shard has been fully compacted. Marking shard {} as compacted in Cassandra", targetShard );
@@ -268,7 +267,12 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
//Overwrite our shard index with a newly created one that has been marked as compacted
Shard compactedShard = new Shard( targetShard.getShardIndex(), timeService.getCurrentTime(), true );
final MutationBatch updateMark = edgeShardSerialization.writeShardMeta( scope, compactedShard, edgeMeta );
- HystrixCassandra.async( updateMark );
+ try {
+ updateMark.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to casandra", e );
+ }
resultBuilder.withCompactedShard( compactedShard );
}
@@ -530,40 +534,6 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
- /**
- * Create a thread pool that will reject work if our audit tasks become overwhelmed
- */
- private final class MaxSizeThreadPool extends ThreadPoolExecutor {
-
- public MaxSizeThreadPool( final int workerSize, final int queueLength ) {
- super( 1, workerSize, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>( queueLength ),
- new CompactionThreadFactory(), new RejectionLogger() );
- }
- }
-
-
- private final class CompactionThreadFactory implements ThreadFactory {
-
- private final AtomicLong threadCounter = new AtomicLong();
-
-
- @Override
- public Thread newThread( final Runnable r ) {
- final long newValue = threadCounter.incrementAndGet();
-
- return new Thread( r, "Graph-Shard-Compaction-" + newValue );
- }
- }
-
-
- private final class RejectionLogger implements RejectedExecutionHandler {
-
-
- @Override
- public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- LOG.warn( "Audit queue full, rejecting audit task {}", r );
- }
- }
public static final class CompactionResult {
[05/50] incubator-usergrid git commit: Added getAllEntityFromField to
the entity manager so we can expose it in the service tier.
Posted by to...@apache.org.
Added getAllEntityFromField to the entity manager so we can expose it in the service tier.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d4eb288c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d4eb288c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d4eb288c
Branch: refs/heads/USERGRID-405
Commit: d4eb288c8a04bf73e79b64d1a06d12fca11260ee
Parents: acde0e4
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 11 09:57:11 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 11 09:57:11 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 7 ++++++-
.../org/apache/usergrid/persistence/EntityManager.java | 9 ++++++++-
.../persistence/cassandra/EntityManagerImpl.java | 11 +++++++++++
3 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4eb288c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..4b62fbe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -33,6 +33,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
+import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -500,7 +501,6 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
-
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
if ( logger.isDebugEnabled() ) {
@@ -729,6 +729,11 @@ public class CpEntityManager implements EntityManager {
create( entityType, null );
}
+ @Override
+ public Observable<FieldSet> getAllEntityFromFields(CollectionScope collectionScope,Collection<Field> fields){
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ return ecm.getAllEntities( fields );
+ }
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4eb288c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 4054d77..459032c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -17,6 +17,8 @@
package org.apache.usergrid.persistence;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.index.query.Query;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -25,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import me.prettyprint.hector.api.mutation.Mutator;
+import rx.Observable;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
@@ -34,6 +37,7 @@ import org.apache.usergrid.persistence.entities.Role;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query.Level;
+import org.apache.usergrid.persistence.model.field.Field;
/**
@@ -712,4 +716,7 @@ public interface EntityManager {
* Get health status of application's index.
*/
public Health getIndexHealth();
-}
+
+ public Observable<FieldSet> getAllEntityFromFields(CollectionScope collectionScope,Collection<Field> fields);
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4eb288c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index be43920..adf106a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -69,6 +69,8 @@ import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.CounterUtils.AggregateCounterSelection;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.entities.Event;
import org.apache.usergrid.persistence.entities.Group;
@@ -83,6 +85,7 @@ import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.CounterFilterPredicate;
import org.apache.usergrid.persistence.index.query.Query.Level;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.ClassUtils;
import org.apache.usergrid.utils.CompositeUtils;
@@ -107,6 +110,7 @@ import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.MultigetSliceCounterQuery;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.SliceCounterQuery;
+import rx.Observable;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
@@ -2947,4 +2951,11 @@ public class EntityManagerImpl implements EntityManager {
public Health getIndexHealth() {
return Health.GREEN; // no good way to assess index status using old-school entity manager
}
+
+
+ @Override
+ public Observable<FieldSet> getAllEntityFromFields( final CollectionScope collectionScope,
+ final Collection<Field> fields ) {
+ throw new UnsupportedOperationException( "Not supported." );
+ }
}
[11/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/001f3f85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/001f3f85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/001f3f85
Branch: refs/heads/USERGRID-405
Commit: 001f3f85d7b943b1ddb58f8191e97e16f92f330c
Parents: 67fa78e 6263594
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:45:29 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:45:29 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 10 ----
.../batch/service/JobSchedulerService.java | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 4 --
.../corepersistence/CpRelationManager.java | 14 -----
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 23 +-------
.../cassandra/RelationManagerImpl.java | 58 +++-----------------
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
stack/pom.xml | 43 ---------------
stack/rest/pom.xml | 4 --
.../org/apache/usergrid/rest/RootResource.java | 6 --
stack/test-utils/pom.xml | 5 --
15 files changed, 10 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
[29/50] incubator-usergrid git commit: [USERGRID-396] Everywhere that
an alias is used, we now do the repair entity retrieval.
Posted by to...@apache.org.
[USERGRID-396] Everywhere that an alias is used, we now do the repair entity retrieval.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c0102e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c0102e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c0102e6
Branch: refs/heads/USERGRID-405
Commit: 4c0102e678d67e235d877654f60ec9947e08c160
Parents: a6929ab
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 12 15:40:42 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 12 15:40:42 2015 -0700
----------------------------------------------------------------------
.../services/AbstractCollectionService.java | 24 +++++-------
.../services/AbstractConnectionsService.java | 39 +++++++-------------
2 files changed, 22 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c0102e6/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 26eb6b9..0820737 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -148,7 +148,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- EntityRef entity = em.getAlias( getEntityType(), name );
+ Entity entity = em.getAllEntityFromFields( getEntityType(), name );
if ( entity == null ) {
logger.info( "miss on entityType: {} with name: {}", getEntityType(), name );
@@ -165,8 +165,7 @@ public class AbstractCollectionService extends AbstractService {
}
if ( !context.moreParameters() ) {
- entity = em.get( entity );
- entity = importEntity( context, ( Entity ) entity );
+ entity = importEntity( context, entity );
}
checkPermissionsForEntity( context, entity );
@@ -291,9 +290,9 @@ public class AbstractCollectionService extends AbstractService {
return getItemByName( context, name );
}
- EntityRef ref = em.getAlias( getEntityType(), name );
- Entity entity;
- if ( ref == null ) {
+ // EntityRef ref = em.getAlias( getEntityType(), name );
+ Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ if ( entity == null ) {
// null entity ref means we tried to put a non-existing entity
// before we create a new entity for it, we should check for permission
checkPermissionsForCollection(context);
@@ -305,7 +304,6 @@ public class AbstractCollectionService extends AbstractService {
entity = em.create( getEntityType(), properties );
}
else {
- entity = em.get( ref );
entity = importEntity( context, entity );
checkPermissionsForEntity( context, entity );
updateEntity( context, entity );
@@ -437,12 +435,12 @@ public class AbstractCollectionService extends AbstractService {
return super.postItemByName( context, name );
}
- EntityRef ref = em.getAlias( getEntityType(), name );
- if ( ref == null ) {
+ Entity entity = em.getAllEntityFromFields( getEntityType(), name );
+ if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
- return postItemById( context, ref.getUuid() );
+ return postItemById( context, entity.getUuid() );
}
@@ -491,11 +489,7 @@ public class AbstractCollectionService extends AbstractService {
return getItemByName( context, name );
}
- EntityRef ref = em.getAlias( getEntityType(), name );
- if ( ref == null ) {
- throw new ServiceResourceNotFoundException( context );
- }
- Entity entity = em.get( ref );
+ Entity entity = em.getAllEntityFromFields(getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c0102e6/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index 72be0a2..8a95e5c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -172,13 +172,13 @@ public class AbstractConnectionsService extends AbstractService {
Results r = null;
if ( connecting() ) {
- r = em.getConnectingEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
+ r = em.getConnectingEntities(
+ new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
context.getCollectionName(), null, Level.ALL_PROPERTIES );
}
else {
- r = em.getConnectedEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
+ r = em.getConnectedEntities(
+ new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
context.getCollectionName(), null, Level.ALL_PROPERTIES );
}
@@ -239,7 +239,7 @@ public class AbstractConnectionsService extends AbstractService {
throw new ServiceResourceNotFoundException( context );
}
- if ( results.size() == 1 && !em.isConnectionMember(
+ if ( results.size() == 1 && !em.isConnectionMember(
context.getOwner(), context.getCollectionName(), results.getEntity() ) ) {
throw new ServiceResourceNotFoundException( context );
}
@@ -263,14 +263,9 @@ public class AbstractConnectionsService extends AbstractService {
nameProperty = "name";
}
- EntityRef ref = em.getAlias( query.getEntityType(), name );
- if ( ref == null ) {
- return null;
- }
-
//TODO T.N. USERGRID-1919 actually validate this is connected
- Entity entity = em.get( ref );
+ Entity entity = em.getAllEntityFromFields( query.getEntityType(),name );
if ( entity == null ) {
return null;
}
@@ -294,7 +289,7 @@ public class AbstractConnectionsService extends AbstractService {
}
// query.setLimit( count );
- // usergrid-2389: User defined limit in the query is ignored. Fixed it by following
+ // usergrid-2389: User defined limit in the query is ignored. Fixed it by following
// same style in AstractCollectionService
query.setLimit( query.getLimit( count ) );
query.setResultsLevel( level );
@@ -309,11 +304,11 @@ public class AbstractConnectionsService extends AbstractService {
else {
// r = em.getConnectingEntities( context.getOwner().getUuid(), query.getConnectionType(),
// query.getEntityType(), level );
- // usergrid-2389: User defined limit in the query is ignored. Fixed it by adding
+ // usergrid-2389: User defined limit in the query is ignored. Fixed it by adding
// the limit to the method parameter downstream.
- r = em.getConnectingEntities(
- new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
- query.getConnectionType(),query.getEntityType(), level , query.getLimit());
+ r = em.getConnectingEntities(
+ new SimpleEntityRef( context.getOwner().getType(), context.getOwner().getUuid()),
+ query.getConnectionType(),query.getEntityType(), level , query.getLimit());
}
}
else {
@@ -366,11 +361,7 @@ public class AbstractConnectionsService extends AbstractService {
if ( query.containsSingleNameOrEmailIdentifier() ) {
String name = query.getSingleNameOrEmailIdentifier();
- EntityRef ref = em.getAlias( query.getEntityType(), name );
- if ( ref == null ) {
- throw new ServiceResourceNotFoundException( context );
- }
- entity = em.get( ref );
+ entity = em.getAllEntityFromFields( query.getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
@@ -481,11 +472,7 @@ public class AbstractConnectionsService extends AbstractService {
nameProperty = "name";
}
- EntityRef ref = em.getAlias( query.getEntityType(), name );
- if ( ref == null ) {
- throw new ServiceResourceNotFoundException( context );
- }
- Entity entity = em.get( ref );
+ Entity entity = em.getAllEntityFromFields( query.getEntityType(), name );
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
}
[28/50] incubator-usergrid git commit: [USERGRID-396] Added ability
to short circuit if not unique id's are found.
Posted by to...@apache.org.
[USERGRID-396] Added ability to short circuit if not unique id's are found.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a6929ab2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a6929ab2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a6929ab2
Branch: refs/heads/USERGRID-405
Commit: a6929ab249e88d85c6025660f44392ed2b9c93a7
Parents: c872b69
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 12 12:07:46 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 12 12:07:46 2015 -0700
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6929ab2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index d95183d..7db34aa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -287,6 +287,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
} );
}
+
+ /**
+ * Retrieves all entities that correspond to each field given in the Collection.
+ * @param fields
+ * @return
+ */
@Override
public Observable<FieldSet> getAllEntities(final Collection<Field> fields) {
return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>() {
@@ -296,8 +302,14 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UUID startTime = UUIDGenerator.newTimeUUID();
+ //Get back set of unique values that correspond to collection of fields
UniqueValueSet set = uniqueValueSerializationStrategy.load( collectionScope, fields );
+ //Short circut if we don't have any uniqueValues from the given fields.
+ if(!set.iterator().hasNext()){
+ return new FieldSetImpl( 0 );
+ }
+
//loop through each field, and construct an entity load
List<Id> entityIds = new ArrayList<>(fields.size());
@@ -311,6 +323,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
uniqueValues.add(value);
}
+ //Load a entity for each entityId we retrieved.
final EntitySet entitySet = entitySerializationStrategy.load(collectionScope, entityIds, startTime);
//now loop through and ensure the entities are there.
@@ -334,7 +347,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
- //fire and forget, we don't care. We'll repair it again if we have to
+ //TODO: explore making this an Async process
+ //We'll repair it again if we have to
deleteBatch.execute();
return response;
[15/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9630fcf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9630fcf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9630fcf2
Branch: refs/heads/USERGRID-405
Commit: 9630fcf2aaee9e2e56878852d1fad1cf985c6343
Parents: 615e3fc aeb15e9
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 15:23:33 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 15:23:33 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[02/50] incubator-usergrid git commit: add timer and meter to update
entity
Posted by to...@apache.org.
add timer and meter to update entity
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/06fe6097
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/06fe6097
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/06fe6097
Branch: refs/heads/USERGRID-405
Commit: 06fe6097cb854fa35e9071c87efe8cb7ec8f9f88
Parents: 56c5701
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 08:49:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 08:49:13 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/06fe6097/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1663756..26eff4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -33,6 +33,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
+import com.codahale.metrics.Meter;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,6 +207,8 @@ public class CpEntityManager implements EntityManager {
private Timer entGetEntityCountersTimer;
private Timer esIndexEntityCollectionTimer;
private Timer entRevokeRolePermissionsTimer;
+ private Timer updateEntityTimer;
+ private Meter updateEntityMeter;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -267,6 +270,9 @@ public class CpEntityManager implements EntityManager {
this.entRevokeRolePermissionsTimer =
this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+ this.updateEntityMeter =this.metricsFactory.getMeter(CpEntityManager.class,"cp.entity.update.meter");
+ this.updateEntityTimer =this.metricsFactory.getTimer(CpEntityManager.class, "cp.entity.update.timer");
+
// set to false for now
this.skipAggregateCounters = false;
@@ -562,6 +568,9 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull(appId,"app scope should never be null");
// first, update entity index in its own collection scope
+ updateEntityMeter.mark();
+ Timer.Context timer = updateEntityTimer.time();
+
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -618,6 +627,7 @@ public class CpEntityManager implements EntityManager {
// update in all containing collections and connection indexes
CpRelationManager rm = ( CpRelationManager ) getRelationManager( entity );
rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
+ timer.stop();
}
[19/50] incubator-usergrid git commit: Changes nodes to 2 per shards
with 1 replica.
Posted by to...@apache.org.
Changes nodes to 2 per shards with 1 replica.
Changes ES system properties
Adds remote debugging to tomcat by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fa0705e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fa0705e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fa0705e7
Branch: refs/heads/USERGRID-405
Commit: fa0705e76b001be5e873cb0b1dd1c879d2e7b309
Parents: c594346
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:04:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:04:06 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 20 +++++++++++---------
.../dist/init_instance/install_elasticsearch.sh | 10 ++++------
.../main/groovy/configure_elasticsearch.groovy | 11 +++++++++++
.../src/main/groovy/configure_usergrid.groovy | 2 +-
4 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index cad4e74..ff74851 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -1,14 +1,14 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. The ASF licenses this file to You
# under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,10 +35,10 @@ apt-get -y --force-yes install ${PKGS}
# Install AWS Java SDK and get it into the Groovy classpath
curl http://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip > /tmp/aws-sdk-java.zip
cd /usr/share/
-unzip /tmp/aws-sdk-java.zip
+unzip /tmp/aws-sdk-java.zip
mkdir -p /home/ubuntu/.groovy/lib
cp /usr/share/aws-java-sdk-*/third-party/*/*.jar /home/ubuntu/.groovy/lib
-cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
ln -s /home/ubuntu/.groovy /root/.groovy
# Build environment for Groovy scripts
@@ -55,7 +55,7 @@ groovy tag_instance.groovy -BUILD-IN-PROGRESS
chmod +x /usr/share/usergrid/update.sh
cd /usr/share/usergrid/init_instance
-./install_oraclejdk.sh
+./install_oraclejdk.sh
cd /usr/share/usergrid/init_instance
./install_yourkit.sh
@@ -119,6 +119,8 @@ sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount
#Append our java opts for secret key
echo "JAVA_OPTS=\"\${JAVA_OPTS} -DAWS_SECRET_KEY=${AWS_SECRET_KEY} -DAWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY}\"" >> /etc/default/tomcat7
+echo "JAVA_OPTS=\"\${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000\"" >> /etc/default/tomcat7
+
ulimit -n $NOFILE
@@ -168,7 +170,7 @@ kernel.shmall = 4294967296
######
EOF
-# wait for enough Cassandra nodes then delpoy and configure Usergrid
+# wait for enough Cassandra nodes then delpoy and configure Usergrid
cd /usr/share/usergrid/scripts
groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
@@ -182,8 +184,8 @@ chown -R tomcat7 /usr/share/usergrid/webapps
chown -R tomcat7 /var/lib/tomcat7/webapps
# configure usergrid
-mkdir -p /usr/share/tomcat7/lib
-groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
+mkdir -p /usr/share/tomcat7/lib
+groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index b2018e1..b488320 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -80,8 +80,10 @@ cat >> /etc/default/elasticsearch << EOF
ES_HEAP_SIZE=${ES_HEAP_SIZE}
MAX_OPEN_FILES=65535
MAX_MAP_COUNT=262144
-#MAX_LOCKED_MEMORY=unlimited
+MAX_LOCKED_MEMORY=unlimited
JAVA_HOME=/usr/lib/jvm/jdk1.7.0
+ES_HEAP_NEWSIZE=4g
+ES_JAVA_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:SurvivorRatio=4 -Xloggc:/mnt/raid/elasticsearch/jvm"
EOF
#Set it because Matt says so
@@ -106,15 +108,11 @@ pushd /usr/share/elasticsearch/bin
#Install bigdesk
-./plugin --install lukas-vlcek/bigdesk
-
-./plugin --install mobz/elasticsearch-head
-
./plugin -install royrusso/elasticsearch-HQ
./plugin -install karmi/elasticsearch-paramedic
-./plugin -install xyu/elasticsearch-whatson/0.1.3
+./plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.1
popd
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 19c969d..38b032f 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -26,6 +26,9 @@
//
+String accessKey = (String)System.getenv().get("AWS_ACCESS_KEY")
+String secretKey = (String)System.getenv().get("AWS_SECRET_KEY")
+
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def clusterName = (String)System.getenv().get("ES_CLUSTER_NAME")
@@ -153,6 +156,14 @@ index.indexing.slowlog.threshold.index.info: 5s
index.indexing.slowlog.threshold.index.debug: 2s
index.indexing.slowlog.threshold.index.trace: 500ms
+########
+# AWS PLUGIM
+##########
+
+cloud.aws.access_key: ${accessKey}
+cloud.aws.secret_key: ${secretKey}
+
+
"""
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 9274430..8af5884 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -44,7 +44,7 @@ def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
def numEsNodes = Integer.parseInt(System.getenv().get("ES_NUM_SERVERS"))
//Override number of shards. Set it to 2x the cluster size
-def esShards = numEsNodes;
+def esShards = numEsNodes*2;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
[26/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/19c6ad0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/19c6ad0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/19c6ad0b
Branch: refs/heads/USERGRID-405
Commit: 19c6ad0bc01d79719079985b4930956c1ed17b48
Parents: c319d07 e8dc17d
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 19:34:13 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 19:34:13 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/core/metrics/MetricsFactoryImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[34/50] incubator-usergrid git commit: add replay strategy
Posted by to...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a1b557d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a1b557d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a1b557d0
Branch: refs/heads/USERGRID-405
Commit: a1b557d073fd8dbe4c20f949ce7497db01d7cea5
Parents: fde59b7
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 11:25:52 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 11:25:52 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 167 ++++++++++++-------
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../persistence/core/astyanax/CassandraFig.java | 4 +-
3 files changed, 111 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 1c30e75..5f96b92 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -25,7 +25,14 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.hystrix.Hystrix;
+import com.netflix.hystrix.HystrixCommand;
+import com.netflix.hystrix.HystrixCommandGroupKey;
+import com.netflix.hystrix.HystrixThreadPoolProperties;
import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,12 +73,14 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
protected final SerializationFig serializationFig;
protected final Keyspace keyspace;
+ private final CassandraConfig cassandraFig;
@Inject
public WriteUniqueVerify( final UniqueValueSerializationStrategy uniqueValueSerializiationStrategy,
- final SerializationFig serializationFig, final Keyspace keyspace ) {
+ final SerializationFig serializationFig, final Keyspace keyspace, final CassandraConfig cassandraFig ) {
this.keyspace = keyspace;
+ this.cassandraFig = cassandraFig;
Preconditions.checkNotNull( uniqueValueSerializiationStrategy, "uniqueValueSerializationStrategy is required" );
Preconditions.checkNotNull( serializationFig, "serializationFig is required" );
@@ -88,100 +97,134 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final MvccEntity mvccEntity = ioevent.getEvent();
- final Id entityId = mvccEntity.getId();
-
- final UUID entityVersion = mvccEntity.getVersion();
-
-
final Entity entity = mvccEntity.getEntity().get();
-
final CollectionScope scope = ioevent.getEntityCollection();
// use simple thread pool to verify fields in parallel
+ ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,keyspace,serializationFig,cassandraFig,scope,entity);
+ Map<String,Field> uniquenessViolations = cmd.execute();
+ //We have violations, throw an exception
+ if ( !uniquenessViolations.isEmpty() ) {
+ throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
+ }
+ }
- final Collection<Field> entityFields = entity.getFields();
+ private static class ConsistentReplayCommand extends HystrixCommand<Map<String,Field>>{
+
+ private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ private final Keyspace keySpace;
+ private final SerializationFig serializationFig;
+ private final CassandraConfig fig;
+ private final CollectionScope scope;
+ private final Entity entity;
+
+ public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy,Keyspace keySpace, SerializationFig serializationFig, CassandraConfig fig,CollectionScope scope, Entity entity){
+ super(REPLAY_GROUP);
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.keySpace = keySpace;
+ this.serializationFig = serializationFig;
+ this.fig = fig;
+ this.scope = scope;
+ this.entity = entity;
+ }
- //allocate our max size, worst case
- final List<Field> uniqueFields = new ArrayList<>( entityFields.size() );
+ @Override
+ protected Map<String, Field> run() throws Exception {
+ return executeStrategy(fig.getReadCL());
+ }
- final MutationBatch batch = keyspace.prepareMutationBatch();
- //
- // Construct all the functions for verifying we're unique
- //
- for ( final Field field : entityFields ) {
+ @Override
+ protected Map<String, Field> getFallback() {
+ return executeStrategy(fig.getConsistentReadCL());
+ }
- // if it's unique, create a function to validate it and add it to the list of
- // concurrent validations
- if ( field.isUnique() ) {
+ public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
+ Collection<Field> entityFields = entity.getFields();
+ //allocate our max size, worst case
+ final List<Field> uniqueFields = new ArrayList<Field>( entityFields.size() );
+ //now get the set of fields back
+ final UniqueValueSet uniqueValues;
+ //todo add consistencylevel and read back if fail using
+ try {
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, entityId, entityVersion );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to read from cassandra", e );
+ }
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
- batch.mergeShallow( mb );
+ final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
- uniqueFields.add( field );
- }
- }
+ //loop through each field that was unique
+ for ( final Field field : uniqueFields ) {
- //short circuit nothing to do
- if ( uniqueFields.size() == 0 ) {
- return;
- }
+ final UniqueValue uniqueValue = uniqueValues.getValue( field.getName() );
+ if ( uniqueValue == null ) {
+ throw new RuntimeException(
+ String.format( "Could not retrieve unique value for field %s, unable to verify",
+ field.getName() ) );
+ }
- //perform the write
- try {
- batch.execute();
- }
- catch ( ConnectionException ex ) {
- throw new RuntimeException( "Unable to write to cassandra", ex );
- }
+ final Id returnedEntityId = uniqueValue.getEntityId();
- //now get the set of fields back
- final UniqueValueSet uniqueValues;
- try {
- uniqueValues = uniqueValueStrat.load( scope, uniqueFields );
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to read from cassandra", e );
- }
+ if ( !entity.getId().equals(returnedEntityId) ) {
+ uniquenessViolations.put( field.getName(), field );
+ }
+ }
+ final MutationBatch batch = keySpace.prepareMutationBatch();
+ //
+ // Construct all the functions for verifying we're unique
+ //
+ for ( final Field field : entity.getFields() ) {
+ // if it's unique, create a function to validate it and add it to the list of
+ // concurrent validations
+ if ( field.isUnique() ) {
- final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, entity.getId(), entity.getVersion() );
- //loop through each field that was unique
- for ( final Field field : uniqueFields ) {
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueSerializationStrategy.write( scope, written, serializationFig.getTimeout() );
- final UniqueValue uniqueValue = uniqueValues.getValue( field.getName() );
+ batch.mergeShallow( mb );
- if ( uniqueValue == null ) {
- throw new RuntimeException(
- String.format( "Could not retrieve unique value for field %s, unable to verify",
- field.getName() ) );
- }
+ uniqueFields.add(field);
+ }
+ }
- final Id returnedEntityId = uniqueValue.getEntityId();
+ //short circuit nothing to do
+ if ( uniqueFields.size() == 0 ) {
+ return uniquenessViolations ;
+ }
- if ( !entityId.equals( returnedEntityId ) ) {
- uniquenessViolations.put( field.getName(), field );
+ //perform the write
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException ex ) {
+ throw new RuntimeException( "Unable to write to cassandra", ex );
}
- }
-
- //We have violations, throw an exception
- if ( !uniquenessViolations.isEmpty() ) {
- throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
+ return uniquenessViolations;
}
}
+
+ /**
+ * Command group used for realtime user commands
+ */
+ public static final HystrixCommand.Setter
+ REPLAY_GROUP = HystrixCommand.Setter.withGroupKey(
+ HystrixCommandGroupKey.Factory.asKey( "user" ) ).andThreadPoolPropertiesDefaults(
+ HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 4ebfd24..51cb198 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.persistence.collection.mvcc.stage.write;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Rule;
import org.junit.Test;
@@ -64,7 +65,8 @@ public class WriteUniqueVerifyTest {
@Inject
private SerializationFig fig;
-
+ @Inject
+ private CassandraConfig cassandraConfig;
@Test
@@ -81,7 +83,7 @@ public class WriteUniqueVerifyTest {
final MvccEntity mvccEntity = fromEntity( entity );
// run the stage
- WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace );
+ WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace,cassandraConfig );
newStage.call(
new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index 66a716c..a907702 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -79,11 +79,11 @@ public interface CassandraFig extends GuicyFig {
@Default( "false" )
boolean isEmbedded();
- @Default("CL_ONE")
+ @Default("CL_LOCAL_ONE")
@Key(READ_CL)
String getReadCL();
- @Default("CL_LOCAL_ONE")
+ @Default("CL_LOCAL_QUORUM")
@Key(READ_CONSISTENT_CL)
String getConsistentReadCL();
[10/50] incubator-usergrid git commit: Fixes issue with Guice
transitive dependency
Posted by to...@apache.org.
Fixes issue with Guice transitive dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/67fa78ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/67fa78ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/67fa78ef
Branch: refs/heads/USERGRID-405
Commit: 67fa78efd6df6bd365972c7ed40a1d16dd8caac6
Parents: e70391f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:45:15 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:45:15 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/CoreModule.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67fa78ef/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 82beb8d..3230faa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -77,8 +77,8 @@ public class CoreModule extends AbstractModule {
bind(BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
- install(new MapModule());
-// install(new QueueModule()); TODO, re-enable when index module doesn't depend on queue
+// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
+// install(new QueueModule());
bind(ManagerCache.class).to( CpManagerCache.class );
[46/50] incubator-usergrid git commit: merge from two-o
Posted by to...@apache.org.
merge from two-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b53cb07f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b53cb07f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b53cb07f
Branch: refs/heads/USERGRID-405
Commit: b53cb07f376fcdff8ce0b3226bbb07843f12d4e9
Parents: 0334119 b0a07ba
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 16 17:35:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 16 17:35:24 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 49 ++++++++++-
.../usergrid/persistence/EntityManager.java | 10 ++-
.../cassandra/EntityManagerImpl.java | 8 +-
.../usergrid/persistence/EntityManagerIT.java | 1 +
.../collection/EntityCollectionManager.java | 13 ++-
.../persistence/collection/FieldSet.java | 48 ++++++++++
.../cache/CachedEntityCollectionManager.java | 11 ++-
.../impl/EntityCollectionManagerImpl.java | 92 +++++++++++++++++++-
.../serialization/impl/MutableFieldSet.java | 63 ++++++++++++++
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 72 ++++++++++++++-
.../services/AbstractCollectionService.java | 40 +++------
.../services/AbstractConnectionsService.java | 39 +++------
13 files changed, 374 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e2b7bd1,e5223f0..789e640
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -33,7 -33,7 +33,8 @@@ import java.util.TreeMap
import java.util.TreeSet;
import java.util.UUID;
+import com.codahale.metrics.Meter;
+ import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -206,8 -207,7 +207,9 @@@ public class CpEntityManager implement
private Timer entGetEntityCountersTimer;
private Timer esIndexEntityCollectionTimer;
private Timer entRevokeRolePermissionsTimer;
+ private Timer entGetRepairedEntityTimer;
+ private Timer updateEntityTimer;
+ private Meter updateEntityMeter;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@@ -268,10 -268,9 +270,12 @@@
.getTimer( CpEntityManager.class, "cp.entity.es.index.entity.to.collection.timer" );
this.entRevokeRolePermissionsTimer =
this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+ this.entGetRepairedEntityTimer = this.metricsFactory
+ .getTimer( CpEntityManager.class, "get.repaired.entity.timer" );
+ this.updateEntityMeter =this.metricsFactory.getMeter(CpEntityManager.class,"cp.entity.update.meter");
+ this.updateEntityTimer =this.metricsFactory.getTimer(CpEntityManager.class, "cp.entity.update.timer");
+
// set to false for now
this.skipAggregateCounters = false;
@@@ -566,12 -577,8 +581,11 @@@
Preconditions.checkNotNull(appId,"app scope should never be null");
// first, update entity index in its own collection scope
+ updateEntityMeter.mark();
+ Timer.Context timer = updateEntityTimer.time();
+
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
-
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
if ( logger.isDebugEnabled() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 92b07f0,3c89280..6ba4513
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -70,11 -61,12 +72,14 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
+ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
+import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
[47/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
USERGRID-405
Posted by to...@apache.org.
Merge branch 'two-dot-o' into USERGRID-405
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7b5d2224
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7b5d2224
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7b5d2224
Branch: refs/heads/USERGRID-405
Commit: 7b5d2224512f0043336acbd9b9323e1e32edf888
Parents: 5c7a5f8 b53cb07
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 17 11:36:08 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 17 11:36:08 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 2 +
.../dist/init_instance/install_elasticsearch.sh | 22 +-
.../main/groovy/configure_elasticsearch.groovy | 80 ++--
.../src/main/groovy/configure_usergrid.groovy | 12 +-
stack/awscluster/ugcluster-cf.json | 466 +++++++++++++------
stack/core/pom.xml | 10 -
.../batch/service/JobSchedulerService.java | 5 -
.../usergrid/corepersistence/CoreModule.java | 36 +-
.../corepersistence/CpEntityManager.java | 64 ++-
.../corepersistence/CpEntityManagerFactory.java | 4 -
.../corepersistence/CpRelationManager.java | 14 -
.../events/EntityDeletedHandler.java | 58 ++-
.../events/EntityVersionCreatedHandler.java | 60 ++-
.../events/EntityVersionDeletedHandler.java | 53 ++-
.../results/FilteringLoader.java | 2 +-
.../usergrid/persistence/EntityManager.java | 10 +-
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 31 +-
.../cassandra/RelationManagerImpl.java | 58 +--
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
.../corepersistence/StaleIndexCleanupTest.java | 38 +-
.../usergrid/persistence/EntityManagerIT.java | 1 +
.../PerformanceEntityRebuildIndexTest.java | 2 +
.../collection/EntityCollectionManager.java | 20 +-
.../collection/EntityDeletedFactory.java | 34 --
.../collection/EntityVersionCleanupFactory.java | 35 --
.../collection/EntityVersionCreatedFactory.java | 31 --
.../persistence/collection/FieldSet.java | 48 ++
.../cache/CachedEntityCollectionManager.java | 11 +-
.../collection/guice/CollectionModule.java | 11 +-
.../EntityCollectionManagerFactoryImpl.java | 32 +-
.../impl/EntityCollectionManagerImpl.java | 251 ++++++++--
.../collection/impl/EntityDeletedTask.java | 9 +-
.../impl/EntityVersionCleanupTask.java | 34 +-
.../impl/EntityVersionTaskFactory.java | 65 +++
.../mvcc/stage/write/WriteUniqueVerify.java | 140 +++---
.../UniqueValueSerializationStrategy.java | 12 +
.../serialization/impl/MutableFieldSet.java | 63 +++
.../UniqueValueSerializationStrategyImpl.java | 27 +-
.../collection/util/EntityUtils.java | 4 +-
.../collection/EntityCollectionManagerIT.java | 68 +++
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../persistence/core/future/BetterFuture.java | 43 +-
.../core/metrics/MetricsFactory.java | 9 +
.../core/metrics/MetricsFactoryImpl.java | 121 +++--
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 7 +-
.../astyanax/MultiRowColumnIteratorTest.java | 7 +-
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 ++++++++++--
.../usergrid/persistence/map/MapManager.java | 14 +-
.../persistence/map/impl/MapManagerImpl.java | 8 +
.../persistence/map/impl/MapSerialization.java | 9 +
.../map/impl/MapSerializationImpl.java | 130 +++++-
.../persistence/map/MapManagerTest.java | 49 +-
stack/corepersistence/queryindex/pom.xml | 17 +-
.../usergrid/persistence/index/EntityIndex.java | 26 +-
.../persistence/index/IndexBufferConsumer.java | 11 +
.../persistence/index/IndexBufferProducer.java | 1 -
.../usergrid/persistence/index/IndexFig.java | 73 ++-
.../index/IndexOperationMessage.java | 115 ++++-
.../persistence/index/guice/IndexModule.java | 7 +
.../persistence/index/guice/QueueProvider.java | 116 +++++
.../persistence/index/impl/BatchRequest.java | 41 ++
.../persistence/index/impl/BufferQueue.java | 68 +++
.../index/impl/BufferQueueInMemoryImpl.java | 108 +++++
.../index/impl/BufferQueueSQSImpl.java | 307 ++++++++++++
.../persistence/index/impl/DeIndexRequest.java | 115 +++++
.../index/impl/EsEntityIndexBatchImpl.java | 50 +-
.../index/impl/EsEntityIndexImpl.java | 176 ++-----
.../index/impl/EsIndexBufferConsumerImpl.java | 286 ++++++++----
.../index/impl/EsIndexBufferProducerImpl.java | 16 +-
.../persistence/index/impl/EsIndexCache.java | 138 +++---
.../persistence/index/impl/IndexRequest.java | 125 +++++
.../index/guice/TestIndexModule.java | 3 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 +++++++
.../impl/EntityConnectionIndexImplTest.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 20 +-
.../persistence/index/impl/EsTestUtils.java | 48 --
.../persistence/queue/QueueManager.java | 4 +-
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 --
.../persistence/queue/guice/QueueModule.java | 17 +-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --
.../persistence/queue/impl/QueueScopeImpl.java | 27 +-
.../queue/impl/SQSQueueManagerImpl.java | 214 +++++----
.../persistence/queue/NoAWSCredsRule.java | 98 ++++
.../persistence/queue/QueueManagerTest.java | 29 +-
stack/pom.xml | 43 --
stack/rest/pom.xml | 4 -
.../org/apache/usergrid/rest/RootResource.java | 6 -
stack/services/pom.xml | 13 +-
.../cassandra/ManagementServiceImpl.java | 9 +-
.../services/AbstractCollectionService.java | 40 +-
.../services/AbstractConnectionsService.java | 39 +-
.../notifications/NotificationsService.java | 50 +-
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
.../notifications/NotifiersServiceIT.java | 6 +
stack/test-utils/pom.xml | 5 -
105 files changed, 3849 insertions(+), 1571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index cf927ce,161037b..5da7067
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@@ -33,21 -30,17 +30,15 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
- import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
- import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
- import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
- import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
-
- import com.google.inject.AbstractModule;
- import com.google.inject.Provider;
- import com.google.inject.TypeLiteral;
- import com.google.inject.multibindings.Multibinder;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.ApplicationContext;
/**
@@@ -76,31 -69,11 +67,31 @@@ public class CoreModule extends Abstra
bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
install( new CommonModule());
- install(new CollectionModule());
- install(new GraphModule());
- install( new IndexModule() );
-// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
-// install(new QueueModule());
+ install( new CollectionModule() {
+ /**
+ * configure our migration data provider for all entities in the system
+ */
+ @Override
+ public void configureMigrationProvider() {
+
+ bind(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){}).to(
+ AllEntitiesInSystemImpl.class );
+ }
+ } );
+ install( new GraphModule() {
+
+ /**
+ * Override the observable that needs to be used for migration
+ */
+ @Override
+ public void configureMigrationProvider() {
+ bind( new TypeLiteral<MigrationDataProvider<GraphNode>>() {} ).to(
+ AllNodesInGraphImpl.class );
+ }
+ } );
+ install(new IndexModule());
- install(new MapModule());
- install(new QueueModule());
++ // install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
++ // install(new QueueModule());
bind(ManagerCache.class).to( CpManagerCache.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 65dacee,01848cb..95ff9dc
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@@ -47,18 -53,18 +54,20 @@@ import rx.schedulers.Schedulers
public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionDeletedHandler.class );
- @Inject
- private SerializationFig serializationFig;
+
+
+
+ private final EntityManagerFactory emf;
@Inject
- private EntityManagerFactory emf;
+ public EntityVersionDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
+
@Override
- public void versionDeleted(
- final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
+ final List<MvccLogEntry> entityVersions ) {
+
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
@@@ -87,19 -89,18 +92,19 @@@
scope.getName()
);
- rx.Observable.from( entityVersions )
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccLogEntry>, List<MvccLogEntry>>() {
+ Observable.from( entityVersions )
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
@Override
- public List<MvccLogEntry> call(List<MvccLogEntry> entityList) {
- for (MvccLogEntry entity : entityList) {
- eibatch.deindex(indexScope, entityId, entity.getVersion());
- }
- eibatch.execute();
- return entityList;
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
+ entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
}
- }).toBlocking().last();
+ } ).doOnNext( new Action1<EntityIndexBatch>() {
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch ) {
+ entityIndexBatch.execute();
+ }
+ } ).toBlocking().last();
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/cache/CachedEntityCollectionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index d5478d4,a73d7a7..68ddab5
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@@ -21,19 -21,17 +21,14 @@@ package org.apache.usergrid.persistence
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
- import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
- import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
- import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
- import org.apache.usergrid.persistence.collection.MvccEntity;
+ import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerFactoryImpl;
--import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.SerializationModule;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 10d85f8,23e375d..ac82181
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@@ -26,25 -25,25 +26,18 @@@ import java.util.concurrent.ExecutionEx
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+ import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
+import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
+import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
--import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
- import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
- import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
- import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
+ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
- import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
- import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
--import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
--import org.apache.usergrid.persistence.core.task.TaskExecutor;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@@ -74,25 -74,22 +67,24 @@@ public class EntityCollectionManagerFac
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final Keyspace keyspace;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
+ private final EntityCacheFig entityCacheFig;
+ private final MetricsFactory metricsFactory;
+ private EntityCacheFig entityCacheFig;
+ private SerializationFig serializationFig;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
.build( new CacheLoader<CollectionScope, EntityCollectionManager>() {
public EntityCollectionManager load( CollectionScope scope ) {
//create the target EM that will perform logic
- final EntityCollectionManager target = new EntityCollectionManagerImpl( writeStart, writeUpdate, writeVerifyUnique,
+ final EntityCollectionManager target = new EntityCollectionManagerImpl(
+ writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope );
- mvccLogEntrySerializationStrategy, keyspace, entityVersionTaskFactory,
- taskExecutor, scope, metricsFactory );
++ mvccLogEntrySerializationStrategy, keyspace, serializationFig,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@@ -113,14 -111,13 +105,12 @@@
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
final EntityCacheFig entityCacheFig,
- final SerializationFig serializationFig) {
+ MetricsFactory metricsFactory) {
this.writeStart = writeStart;
- this.writeUpdate = writeUpdate;
this.writeVerifyUnique = writeVerifyUnique;
this.writeOptimisticVerify = writeOptimisticVerify;
this.writeCommit = writeCommit;
@@@ -131,20 -128,21 +121,18 @@@
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
- this.serializationFig = serializationFig;
+ this.metricsFactory = metricsFactory;
}
-
-
@Override
- public EntityCollectionManager createCollectionManager( CollectionScope collectionScope ) {
- Preconditions.checkNotNull( collectionScope );
- try {
- return ecmCache.get( collectionScope );
- }
- catch ( ExecutionException ee ) {
- throw new RuntimeException( ee );
+ public EntityCollectionManager createCollectionManager(CollectionScope collectionScope) {
+ Preconditions.checkNotNull(collectionScope);
+ try{
+ return ecmCache.get(collectionScope);
+ }catch (ExecutionException ee){
+ throw new RuntimeException(ee);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 19e8937,6ba4513..001e0db
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -35,6 -34,9 +37,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
++import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.collection.guice.Write;
+ import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
+ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@@ -51,8 -51,9 +57,11 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@@ -69,9 -72,14 +80,17 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
++import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
+ import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+ import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@@ -288,6 -368,140 +375,89 @@@ public class EntityCollectionManagerImp
} );
}
+
+ /**
+ * Retrieves all entities that correspond to each field given in the Collection.
+ * @param fields
+ * @return
+ */
+ @Override
+ public Observable<FieldSet> getEntitiesFromFields( final Collection<Field> fields ) {
+ return rx.Observable.just(fields).map( new Func1<Collection<Field>, FieldSet>() {
+ @Override
+ public FieldSet call( Collection<Field> fields ) {
+ try {
+
+ final UUID startTime = UUIDGenerator.newTimeUUID();
+
+ //Get back set of unique values that correspond to collection of fields
+ UniqueValueSet set = uniqueValueSerializationStrategy.load( collectionScope, fields );
+
+ //Short circut if we don't have any uniqueValues from the given fields.
+ if(!set.iterator().hasNext()){
+ return new MutableFieldSet( 0 );
+ }
+
+
+ //loop through each field, and construct an entity load
+ List<Id> entityIds = new ArrayList<>(fields.size());
+ List<UniqueValue> uniqueValues = new ArrayList<>(fields.size());
+
+ for(final Field expectedField: fields) {
+
+ UniqueValue value = set.getValue(expectedField.getName());
+
+ if(value ==null){
+ logger.debug( "Field does not correspond to a unique value" );
+ }
+
+ entityIds.add(value.getEntityId());
+ uniqueValues.add(value);
+ }
+
+ //Load a entity for each entityId we retrieved.
+ final EntitySet entitySet = entitySerializationStrategy.load(collectionScope, entityIds, startTime);
+
+ //now loop through and ensure the entities are there.
+ final MutationBatch deleteBatch = keyspace.prepareMutationBatch();
+
+ final MutableFieldSet response = new MutableFieldSet(fields.size());
+
+ for(final UniqueValue expectedUnique: uniqueValues) {
+ final MvccEntity entity = entitySet.getEntity(expectedUnique.getEntityId());
+
+ //bad unique value, delete this, it's inconsistent
+ if(entity == null || !entity.getEntity().isPresent()){
+ final MutationBatch valueDelete = uniqueValueSerializationStrategy.delete(collectionScope, expectedUnique);
+ deleteBatch.mergeShallow(valueDelete);
+ continue;
+ }
+
+
+ //else add it to our result set
+ response.addEntity(expectedUnique.getField(),entity);
+
+ }
+
+ //TODO: explore making this an Async process
+ //We'll repair it again if we have to
+ deleteBatch.execute();
+
+ return response;
+
+
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Failed to getIdField", e );
+ throw new RuntimeException( e );
+ }
+ }
+ } );
+ }
+
+
+
- @Override
- public Observable<Entity> update( final Entity entity ) {
-
- logger.debug( "Starting update process" );
-
- //do our input validation
- Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
-
- final Id entityId = entity.getId();
-
-
- ValidationUtils.verifyIdentity( entityId );
-
- // create our observable and start the write
- CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
-
-
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
-
-
- final Timer.Context timer = updateTimer.time();
- return observable.map( writeCommit ).doOnNext(new Action1<Entity>() {
- @Override
- public void call(final Entity entity) {
- logger.debug("sending entity to the queue");
-
- //we an update, signal the fix
- taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
-
- taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entity.getId(), entity.getVersion(), false) );
- //TODO T.N Change this to fire a task
- // Observable.from( new CollectionIoEvent<Id>(collectionScope,
- // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
-
-
- }
- }).doOnError(rollback)
- .doOnNext(new Action1<Entity>() {
- @Override
- public void call(Entity entity) {
- updateMeter.mark();
- }
- })
- .doOnCompleted(new Action0() {
- @Override
- public void call() {
- timer.stop();
- }
- });
- }
-
+
// fire the stages
public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
WriteStart writeState ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index dbe58d4,83f165d..5472645
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@@ -22,9 -22,8 +22,8 @@@ import com.google.inject.Inject
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
import org.apache.usergrid.persistence.collection.CollectionScope;
- import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 7c39ef4,65506ec..27c3db8
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@@ -20,9 -20,17 +20,18 @@@ package org.apache.usergrid.persistence
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
+ import com.google.inject.Inject;
+ import com.google.inject.assistedinject.Assisted;
+ import org.apache.usergrid.persistence.collection.MvccEntity;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+ import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+ import org.apache.usergrid.persistence.collection.util.EntityUtils;
+ import org.apache.usergrid.persistence.model.entity.Entity;
+ import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -70,6 -75,7 +79,7 @@@ public class EntityVersionCleanupTask i
private final CollectionScope scope;
private final Id entityId;
private final UUID version;
- private final int numToSkip;
++ private final boolean includeVersion;
@Inject
@@@ -89,6 -100,8 +103,8 @@@
this.scope = scope;
this.entityId = entityId;
this.version = version;
+
- numToSkip = includeVersion? 0: 1;
++ includeVersion = includeVersion;
}
@@@ -117,94 -130,79 +133,92 @@@
@Override
public Void call() throws Exception {
//TODO Refactor this logic into a a class that can be invoked from anywhere
- //load every entity we have history of
- Observable<List<MvccEntity>> deleteFieldsObservable =
- Observable.create(new ObservableIterator<MvccEntity>("deleteColumns") {
- @Override
- protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> entities = entitySerializationStrategy.loadDescendingHistory(
- scope, entityId, version, 1000); // TODO: what fetchsize should we use here?
- return entities;
- }
- })
- //buffer them for efficiency
- .skip(numToSkip)
- .buffer(serializationFig.getBufferSize()).doOnNext(
- new Action1<List<MvccEntity>>() {
- @Override
- public void call(final List<MvccEntity> mvccEntities) {
- final MutationBatch batch = keyspace.prepareMutationBatch();
- final MutationBatch entityBatch = keyspace.prepareMutationBatch();
- final MutationBatch logBatch = keyspace.prepareMutationBatch();
--
- for (MvccEntity mvccEntity : mvccEntities) {
- final UUID entityVersion = mvccEntity.getVersion();
--
- //iterate all unique values
-
- //if the entity is present process the fields
- if(mvccEntity.getEntity().isPresent()) {
- final Entity entity = mvccEntity.getEntity().get();
-
- //remove all unique fields from the index
- for ( final Field field : EntityUtils.getUniqueFields(entity )) {
-
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
- final MutationBatch deleteMutation =
- uniqueValueSerializationStrategy.delete( scope, unique );
- batch.mergeShallow( deleteMutation );
++ //iterate all unique values
+ final BlockingObservable<Long> uniqueValueCleanup =
+ Observable.create( new ObservableIterator<UniqueValue>( "Unique value load" ) {
+ @Override
+ protected Iterator<UniqueValue> getIterator() {
+ return uniqueValueSerializationStrategy.getAllUniqueFields( scope, entityId );
+ }
+ } )
+
+ //skip current versions
+ .skipWhile( new Func1<UniqueValue, Boolean>() {
+ @Override
+ public Boolean call( final UniqueValue uniqueValue ) {
+ return version.equals( uniqueValue.getEntityVersion() );
}
+ } )
+ //buffer our buffer size, then roll them all up in a single batch mutation
+ .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<UniqueValue>>() {
+ @Override
+ public void call( final List<UniqueValue> uniqueValues ) {
+ final MutationBatch uniqueCleanupBatch = keyspace.prepareMutationBatch();
+
+
+ for ( UniqueValue value : uniqueValues ) {
+ uniqueCleanupBatch.mergeShallow( uniqueValueSerializationStrategy.delete( scope, value ) );
}
- final MutationBatch entityDelete = entitySerializationStrategy
- .delete(scope, entityId, mvccEntity.getVersion());
- entityBatch.mergeShallow( entityDelete );
- final MutationBatch logDelete = logEntrySerializationStrategy
- .delete(scope, entityId, version);
- logBatch.mergeShallow(logDelete);
+ try {
+ uniqueCleanupBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute batch mutation", e );
+ }
}
+ } ).subscribeOn( Schedulers.io() ).longCount().toBlocking();
- try {
- batch.execute();
- } catch (ConnectionException e1) {
- throw new RuntimeException("Unable to execute " +
- "unique value " +
- "delete", e1);
- }
- fireEvents(mvccEntities);
- try {
- entityBatch.execute();
- } catch (ConnectionException e) {
- throw new RuntimeException("Unable to delete entities in cleanup", e);
+
+ //start calling the listeners for remove log entries
+ BlockingObservable<Long> versionsDeletedObservable =
+
+ Observable.create( new ObservableIterator<MvccLogEntry>( "Log entry iterator" ) {
+ @Override
+ protected Iterator<MvccLogEntry> getIterator() {
+
+ return new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, version,
+ serializationFig.getBufferSize() );
}
+ } )
+ //skip current version
+ .skipWhile( new Func1<MvccLogEntry, Boolean>() {
+ @Override
+ public Boolean call( final MvccLogEntry mvccLogEntry ) {
+ return version.equals( mvccLogEntry.getVersion() );
+ }
+ } )
+ //buffer them for efficiency
+ .buffer( serializationFig.getBufferSize() ).doOnNext( new Action1<List<MvccLogEntry>>() {
+ @Override
+ public void call( final List<MvccLogEntry> mvccEntities ) {
+
+ fireEvents( mvccEntities );
+
+ final MutationBatch logCleanupBatch = keyspace.prepareMutationBatch();
+
+
+ for ( MvccLogEntry entry : mvccEntities ) {
+ logCleanupBatch.mergeShallow( logEntrySerializationStrategy.delete( scope, entityId, entry.getVersion() ));
+ }
- try {
- logBatch.execute();
- } catch (ConnectionException e) {
- throw new RuntimeException("Unable to delete entities from the log", e);
+ try {
+ logCleanupBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to execute batch mutation", e );
+ }
}
+ } ).subscribeOn( Schedulers.io() ).longCount().toBlocking();
+
+ //wait or this to complete
+ final Long removedCount = uniqueValueCleanup.last();
- }
- }
- );
+ logger.debug( "Removed unique values for {} entities of entity {}", removedCount, entityId );
- final int removedCount = deleteFieldsObservable.count().toBlocking().last();
+ final Long versionCleanupCount = versionsDeletedObservable.last();
- logger.debug("Removed unique values for {} entities of entity {}",removedCount,entityId);
+ logger.debug( "Removed {} previous entity versions of entity {}", versionCleanupCount, entityId );
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 5b7898e,60275d9..d7a6407
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@@ -23,9 -22,9 +23,10 @@@ import java.util.Iterator
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+ import com.netflix.astyanax.model.ConsistencyLevel;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@@ -55,27 -53,23 +56,38 @@@ public interface UniqueValueSerializati
/**
* Load UniqueValue that matches field from collection or null if that value does not exist.
*
- * @param colScope Collection scope in which to look for field name/value
+ * @param collectionScope scope in which to look for field name/value
* @param fields Field name/value to search for
+ *
* @return UniqueValueSet containing fields from the collection that exist in cassandra
+ *
* @throws ConnectionException on error connecting to Cassandra
*/
- public UniqueValueSet load( CollectionScope colScope, Collection<Field> fields ) throws ConnectionException;
+ public UniqueValueSet load( CollectionScope collectionScope, Collection<Field> fields ) throws ConnectionException;
+ /**
- * Load UniqueValue that matches field from collection or null if that value does not exist.
- *
- * @param colScope Collection scope in which to look for field name/value
- * @param consistencyLevel Consistency level of query
- * @param fields Field name/value to search for
- * @return UniqueValueSet containing fields from the collection that exist in cassandra
- * @throws ConnectionException on error connecting to Cassandra
- */
++ * Load UniqueValue that matches field from collection or null if that value does not exist.
++ *
++ * @param colScope Collection scope in which to look for field name/value
++ * @param consistencyLevel Consistency level of query
++ * @param fields Field name/value to search for
++ * @return UniqueValueSet containing fields from the collection that exist in cassandra
++ * @throws ConnectionException on error connecting to Cassandra
++ */
+ public UniqueValueSet load( CollectionScope colScope, ConsistencyLevel consistencyLevel, Collection<Field> fields ) throws ConnectionException;
++
+
+ /**
+ * Loads the currently persisted history of every unique value the entity has held. This will
+ * start from the max version and return values in descending version order. Note that for entities
+ * with more than one unique field, sequential fields can be returned with the same version.
+ * @param collectionScope The scope the entity is stored in
+ * @param entityId
+ * @return
+ */
+ public Iterator<UniqueValue> getAllUniqueFields(CollectionScope collectionScope, Id entityId);
+
+
/**
* Delete the specified Unique Value from Cassandra.
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 7aed8db,47372ae..108f2e8
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@@ -38,16 -36,8 +40,17 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator;
+import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
+import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@@ -78,40 -65,23 +81,43 @@@ public class UniqueValueSerializationSt
private static final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer();
- private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion> CF_UNIQUE_VALUES =
- new MultiTennantColumnFamily<>( "Unique_Values", ROW_KEY_SER,
- ENTITY_VERSION_SER );
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion>
+ CF_UNIQUE_VALUES = new MultiTennantColumnFamily<>( "Unique_Values", ROW_KEY_SER, ENTITY_VERSION_SER );
+
+
++
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ private static final CollectionScopedRowKeySerializer<Id> ENTITY_ROW_KEY_SER =
+ new CollectionScopedRowKeySerializer<>( ID_SER );
+
+
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UniqueFieldEntry>
+ CF_ENTITY_UNIQUE_VALUES =
+ new MultiTennantColumnFamily<>( "Entity_Unique_Values", ENTITY_ROW_KEY_SER, UniqueFieldEntrySerializer.get() );
+
+ public static final int COL_VALUE = 0x0;
+
+
- private final Keyspace keyspace;
+ private final SerializationFig serializationFig;
+ protected final Keyspace keyspace;
- private final CassandraFig cassandraFig;
++ private final CassandraFig cassandraFig;
+
/**
* Construct serialization strategy for keyspace.
*
* @param keyspace Keyspace in which to store Unique Values.
+ * @param serializationFig
*/
@Inject
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final SerializationFig serializationFig ) {
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig) {
- this.cassandraFig = cassandraFig;
++ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig, final SerializationFig serializationFig ) {
this.keyspace = keyspace;
++ this.cassandraFig = cassandraFig;
+ this.serializationFig = serializationFig;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index beff3bd,cf964a3..20edb66
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@@ -2,9 -2,10 +2,10 @@@ package org.apache.usergrid.persistence
import java.util.ArrayList;
- import java.util.Collection;
import java.util.List;
-import java.util.Set;
++import java.util.Collection;
import java.util.UUID;
+ import org.apache.usergrid.persistence.model.field.Field;
import org.apache.commons.lang3.reflect.FieldUtils;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 8f78799,f19d613..246bbca
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@@ -96,7 -126,8 +126,8 @@@ public class GraphManagerImpl implement
final GraphFig graphFig,
final EdgeDeleteListener edgeDeleteListener,
final NodeDeleteListener nodeDeleteListener,
- final ApplicationScope scope) {
- @Assisted final ApplicationScope scope,
++ final ApplicationScope scope,
+ MetricsFactory metricsFactory) {
ValidationUtils.validateApplicationScope( scope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7fccd11,50b994d..27fe705
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@@ -19,8 -19,12 +19,9 @@@
package org.apache.usergrid.persistence.index.guice;
- import org.apache.usergrid.persistence.core.guice.CommonModule;
+ import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+ import org.apache.usergrid.persistence.core.guice.CommonModule;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
public class TestIndexModule extends TestModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index e4043d2,ca9bf79..9cb5297
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@@ -24,10 -24,14 +24,12 @@@ import java.util.*
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
+ import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
+ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b5d2224/stack/pom.xml
----------------------------------------------------------------------
[33/50] incubator-usergrid git commit: Updated queuescope to be name
with no need for application id. We are only using them in our subsystems,
not applications.
Posted by to...@apache.org.
Updated queuescope to be name with no need for application id. We are only using them in our subsystems, not applications.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/361060ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/361060ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/361060ec
Branch: refs/heads/USERGRID-405
Commit: 361060ec3c90488a8c944d2f30fc95310bd41453
Parents: 97634df
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 13 09:10:28 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 13 09:10:28 2015 -0600
----------------------------------------------------------------------
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 29 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
11 files changed, 231 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
new file mode 100644
index 0000000..ea3e046
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.guice;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
+import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
+import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class QueueProvider implements Provider<BufferQueue> {
+
+ private final IndexFig indexFig;
+
+ private final QueueManagerFactory queueManagerFactory;
+ private final MapManagerFactory mapManagerFactory;
+ private final MetricsFactory metricsFactory;
+
+ private BufferQueue bufferQueue;
+
+
+ @Inject
+ public QueueProvider( final IndexFig indexFig, final QueueManagerFactory queueManagerFactory,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ this.indexFig = indexFig;
+
+
+ this.queueManagerFactory = queueManagerFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.metricsFactory = metricsFactory;
+ }
+
+
+ @Override
+ @Singleton
+ public BufferQueue get() {
+ if ( bufferQueue == null ) {
+ bufferQueue = getQueue();
+ }
+
+
+ return bufferQueue;
+ }
+
+
+ private BufferQueue getQueue() {
+ final String value = indexFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new BufferQueueInMemoryImpl( indexFig );
+ case SQS:
+ return new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
index b2b2ec6..cf6bf24 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.queue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-public interface QueueScope extends ApplicationScope {
+public interface QueueScope {
/**
* Get the name of the the map
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
deleted file mode 100644
index 3a508a9..0000000
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.queue;
-
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import java.util.UUID;
-
-/**
- * Created by ApigeeCorporation on 10/23/14.
- */
-public interface QueueScopeFactory {
-
- public QueueScope getScope( UUID id, String queueName);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
index 4e487e1..1b2d6ea 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
@@ -17,15 +17,16 @@
*/
package org.apache.usergrid.persistence.queue.guice;
-import com.google.inject.AbstractModule;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+import org.safehaus.guicyfig.GuicyFigModule;
+
import org.apache.usergrid.persistence.queue.QueueFig;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
import org.apache.usergrid.persistence.queue.impl.SQSQueueManagerImpl;
-import org.safehaus.guicyfig.GuicyFigModule;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
/**
@@ -43,10 +44,8 @@ public class QueueModule extends AbstractModule {
install( new GuicyFigModule( QueueFig.class) );
// create a guice factory for getting our collection manager
- install(new FactoryModuleBuilder()
- .implement(QueueManager.class, SQSQueueManagerImpl.class)
- .build(QueueManagerFactory.class));
- bind( QueueScopeFactory.class ).to( QueueScopeFactoryImpl.class );
+ install( new FactoryModuleBuilder().implement( QueueManager.class, SQSQueueManagerImpl.class )
+ .build( QueueManagerFactory.class ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
deleted file mode 100644
index a145a6e..0000000
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.queue.impl;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.queue.QueueFig;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-
-import java.util.UUID;
-
-/**
- * Returns scope for queues.
- */
-public class QueueScopeFactoryImpl implements QueueScopeFactory {
-
- private QueueFig fig;
-
- @Inject
- public QueueScopeFactoryImpl(QueueFig fig){
- this.fig = fig;
- }
-
- //applicationId is always ManagementApplicationId so not really needed here.
- @Override
- public QueueScope getScope(UUID applicationId, String queueName) {
- return new QueueScopeImpl(new SimpleId(applicationId, fig.getPrefix()), queueName);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
index a928803..381cd8e 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
@@ -24,18 +24,14 @@ import org.apache.usergrid.persistence.queue.QueueScope;
* Created by ApigeeCorporation on 10/3/14.
*/
public class QueueScopeImpl implements QueueScope {
- private final Id owner;
+
private final String name;
- public QueueScopeImpl( final Id owner, final String name ) {
- this.owner = owner;
+ public QueueScopeImpl( final String name ) {
this.name = name;
}
- @Override
- public Id getApplication() {
- return owner;
- }
+
@Override
@@ -57,9 +53,7 @@ public class QueueScopeImpl implements QueueScope {
if ( !name.equals( queueScope.name ) ) {
return false;
}
- if ( !owner.equals( queueScope.owner ) ) {
- return false;
- }
+
return true;
}
@@ -67,17 +61,6 @@ public class QueueScopeImpl implements QueueScope {
@Override
public int hashCode() {
- int result = owner.hashCode();
- result = 31 * result + name.hashCode();
- return result;
- }
-
-
- @Override
- public String toString() {
- return "QueueScopeImpl{" +
- "owner=" + owner +
- ", name='" + name + '\'' +
- '}';
+ return name.hashCode();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index 1fbd9b6..088359a 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -54,6 +54,7 @@ import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -124,7 +125,10 @@ public class SQSQueueManagerImpl implements QueueManager {
private String getName() {
- String name = fig.getPrefix() + "_" + scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
+ String name = fig.getPrefix() + "_" + scope.getName();
+
+ Preconditions.checkArgument(name.length() <= 80, "Your name must be < than 80 characters");
+
return name;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
new file mode 100644
index 0000000..45218c2
--- /dev/null
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.queue;
+
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.amazonaws.AmazonClientException;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule implements TestRule {
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+ try {
+ base.evaluate();
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ if ( t instanceof AmazonClientException ) {
+
+ final AmazonClientException ace = ( AmazonClientException ) t;
+
+ if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+ "could not get aws secret key from system properties" ) ) {
+ //swallow
+ return true;
+ }
+ }
+
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index 935fd16..eecb9e1 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -25,24 +25,24 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
@RunWith( ITRunner.class )
@UseModules( { TestQueueModule.class } )
@@ -53,8 +53,12 @@ public class QueueManagerTest {
@Inject
protected QueueManagerFactory qmf;
- @Inject
- protected QueueScopeFactory queueScopeFactory;
+ /**
+ * Mark tests as ignored if now AWS creds are present
+ */
+ @Rule
+ public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+
protected QueueScope scope;
private QueueManager qm;
@@ -62,21 +66,11 @@ public class QueueManagerTest {
@Before
public void mockApp() {
- this.scope = new QueueScopeImpl( new SimpleId( "application" ), "testQueue" );
+ this.scope = new QueueScopeImpl( "testQueue" );
qm = qmf.getQueueManager(scope);
- queueScopeFactory = new QueueScopeFactoryImpl(queueFig);
}
- @Test
- public void scopeFactory(){
- UUID uuid = UUID.randomUUID();
- String key = "test";
- QueueScope scope =queueScopeFactory.getScope(uuid,key);
- assertEquals(key,scope.getName());
- assertEquals(scope.getApplication().getUuid(),uuid);
- }
- @Ignore("need aws creds")
@Test
public void send() throws IOException,ClassNotFoundException{
String value = "bodytest";
@@ -92,7 +86,6 @@ public class QueueManagerTest {
}
- @Ignore("need aws creds")
@Test
public void sendMore() throws IOException,ClassNotFoundException{
HashMap<String,String> values = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index d34cc22..e45b82a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
import org.apache.usergrid.services.ServiceManager;
import org.apache.usergrid.services.ServiceManagerFactory;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
@@ -47,7 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
public final int MESSAGE_TRANSACTION_TIMEOUT = 25 * 1000;
private final QueueManagerFactory queueManagerFactory;
- private final QueueScopeFactory queueScopeFactory;
public long DEFAULT_SLEEP = 5000;
@@ -84,7 +84,6 @@ public class QueueListener {
this.emf = emf;
this.metricsService = smf.getApplicationContext().getBean( Injector.class ).getInstance(MetricsFactory.class);
this.properties = props;
- this.queueScopeFactory = smf.getApplicationContext().getBean( Injector.class ).getInstance(QueueScopeFactory.class);
}
/**
@@ -146,7 +145,7 @@ public class QueueListener {
com.codahale.metrics.Timer timer = metricsService.getTimer(QueueListener.class, "dequeue");
svcMgr = smf.getServiceManager(smf.getManagementAppId());
LOG.info("getting from queue {} ", queueName);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), queueName);
+ QueueScope queueScope = new QueueScopeImpl( queueName ) {};
QueueManager queueManager = TEST_QUEUE_MANAGER != null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
// run until there are no more active jobs
long runCount = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
index 8277e86..b87904e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
import org.apache.usergrid.services.ServiceManager;
import org.apache.usergrid.services.ServiceManagerFactory;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
@@ -48,7 +49,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class QueueListener {
public final int MESSAGE_TRANSACTION_TIMEOUT = 25 * 1000;
private final QueueManagerFactory queueManagerFactory;
- private final QueueScopeFactory queueScopeFactory;
public long DEFAULT_SLEEP = 5000;
@@ -93,7 +93,6 @@ public abstract class QueueListener {
this.emf = injector.getInstance( EntityManagerFactory.class ); //emf;
this.metricsService = injector.getInstance(MetricsFactory.class);
this.properties = props;
- this.queueScopeFactory = injector.getInstance(QueueScopeFactory.class);
}
@@ -165,7 +164,7 @@ public abstract class QueueListener {
com.codahale.metrics.Timer timer = metricsService.getTimer(QueueListener.class, "dequeue");
svcMgr = smf.getServiceManager(smf.getManagementAppId());
LOG.info("getting from queue {} ", queueName);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), queueName);
+ QueueScope queueScope = new QueueScopeImpl( queueName);
QueueManager queueManager = TEST_QUEUE_MANAGER != null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
// run until there are no more active jobs
long runCount = 0;
[13/50] incubator-usergrid git commit: Fixes issue with cluster
prefix configuration
Posted by to...@apache.org.
Fixes issue with cluster prefix configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aeb15e96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aeb15e96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aeb15e96
Branch: refs/heads/USERGRID-405
Commit: aeb15e96a9a98909c86bbc7e0ad6ba794a21ab1c
Parents: 8fa5c9f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:56:52 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:56:52 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aeb15e96/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 1f7140c..9274430 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -108,7 +108,7 @@ usergrid.write.cl=${writeConsistencyLevel}
elasticsearch.cluster_name=${clusterName}
-elasticsearch.index_prefix=usergrid
+elasticsearch.index_prefix=${stackName}
elasticsearch.hosts=${esnodes}
elasticsearch.port=9300
elasticsearch.number_shards=${esShards}
[14/50] incubator-usergrid git commit: more meters
Posted by to...@apache.org.
more meters
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/615e3fc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/615e3fc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/615e3fc9
Branch: refs/heads/USERGRID-405
Commit: 615e3fc9073530dcfef5774d53b744fe242ba5ea
Parents: 8fa5c9f
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 15:23:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 15:23:16 2015 -0600
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 39 ++++++++++++++++++--
.../graph/impl/GraphManagerImpl.java | 2 +-
2 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615e3fc9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 391e9a5..50b581a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -124,6 +124,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final Timer updateTimer;
private final Timer loadTimer;
private final Timer getLatestTimer;
+ private final Meter deleteMeter;
+ private final Meter getLatestMeter;
+ private final Meter loadMeter;
+ private final Meter updateMeter;
@Inject
@@ -176,9 +180,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.writeTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"write.timer");
this.writeMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "write.meter");
this.deleteTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "delete.timer");
- this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"update.timer");
+ this.deleteMeter= metricsFactory.getMeter(EntityCollectionManagerImpl.class, "delete.meter");
+ this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "update.timer");
+ this.updateMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "update.meter");
this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"load.timer");
+ this.loadMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "load.meter");
this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"latest.timer");
+ this.getLatestMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "latest.meter");
}
@@ -250,12 +258,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return entity.getId();
}
}
- ) .doOnCompleted(new Action0() {
+ )
+ .doOnNext(new Action1<Id>() {
+ @Override
+ public void call(Id id) {
+ deleteMeter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
@Override
public void call() {
timer.stop();
}
- });;
+ });
return o;
}
@@ -281,6 +296,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return Observable.from(entity.getEntity().get());
}
})
+ .doOnNext(new Action1<Entity>() {
+ @Override
+ public void call(Entity entity) {
+ loadMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
@@ -316,6 +337,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}
} )
+ .doOnNext(new Action1<EntitySet>() {
+ @Override
+ public void call(EntitySet entitySet) {
+ loadMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
@@ -382,6 +409,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}).doOnError(rollback)
+ .doOnNext(new Action1<Entity>() {
+ @Override
+ public void call(Entity entity) {
+ updateMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615e3fc9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 53e116d..f19d613 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -166,7 +166,7 @@ public class GraphManagerImpl implements GraphManager {
this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.type.timer");
this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.from.timer");
- this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+ this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.from.meter");
this.getIdTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.idtype.from.timer");
this.getIdTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.from.meter");
[27/50] incubator-usergrid git commit: Fixes issue with duplicate
pool size definition
Posted by to...@apache.org.
Fixes issue with duplicate pool size definition
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/97634dfc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/97634dfc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/97634dfc
Branch: refs/heads/USERGRID-405
Commit: 97634dfc6c5a8e7192d72567d68fd62bfd67a012
Parents: 19c6ad0
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 12 12:14:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 12 12:14:58 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_elasticsearch.groovy | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/97634dfc/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 38b032f..173e4e6 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -87,7 +87,6 @@ threadpool:
bulk:
type: fixed
size: 160
- size: 16
queue_size: 1000
search:
size: 320
[07/50] incubator-usergrid git commit: Fixes transitive dependency
issue with queue module
Posted by to...@apache.org.
Fixes transitive dependency issue with queue module
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/67dd8c02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/67dd8c02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/67dd8c02
Branch: refs/heads/USERGRID-405
Commit: 67dd8c02b7949d6ed212ce0d41e83baa6a74e797
Parents: e58597b
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:11:49 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:11:49 2015 -0600
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/corepersistence/CoreModule.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67dd8c02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7b53d67..82beb8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -78,7 +78,7 @@ public class CoreModule extends AbstractModule {
}
} );
install(new MapModule());
- install(new QueueModule());
+// install(new QueueModule()); TODO, re-enable when index module doesn't depend on queue
bind(ManagerCache.class).to( CpManagerCache.class );
[12/50] incubator-usergrid git commit: Fixes issue with not deleting
from the map on message ack
Posted by to...@apache.org.
Fixes issue with not deleting from the map on message ack
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8fa5c9f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8fa5c9f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8fa5c9f8
Branch: refs/heads/USERGRID-405
Commit: 8fa5c9f8f4b6580eccba820c5fbbfdf9f088ed87
Parents: 001f3f8
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:52:20 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:52:20 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/BufferQueueSQSImpl.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8fa5c9f8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index c6acb36..25c2ba6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -190,7 +190,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//load them into our response
for ( final QueueMessage message : messages ) {
- final String key = message.getBody().toString();
+ final String key = getMessageKey( message );
//now see if the key was there
final String payload = values.get( key );
@@ -238,6 +238,15 @@ public class BufferQueueSQSImpl implements BufferQueue {
List<QueueMessage> toAck = new ArrayList<>( messages.size() );
for ( IndexOperationMessage ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ final String key = getMessageKey(sqsIndexOperationMessage.getMessage());
+
+ //remove it from the map
+ mapManager.delete( key );
+
toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
}
@@ -257,6 +266,9 @@ public class BufferQueueSQSImpl implements BufferQueue {
return mapper.writeValueAsString( o );
}
+ private String getMessageKey(final QueueMessage message){
+ return message.getBody().toString();
+ }
/**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
[38/50] incubator-usergrid git commit: add comment
Posted by to...@apache.org.
add comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8cea08db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8cea08db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8cea08db
Branch: refs/heads/USERGRID-405
Commit: 8cea08db37eafede2dbe3c7a8cd1fd38dc0d548a
Parents: 117c0f3
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:21:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:21:27 2015 -0600
----------------------------------------------------------------------
.../collection/serialization/UniqueValueSerializationStrategy.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8cea08db/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 8d314d8..60275d9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -64,6 +64,7 @@ public interface UniqueValueSerializationStrategy extends Migration {
* Load UniqueValue that matches field from collection or null if that value does not exist.
*
* @param colScope Collection scope in which to look for field name/value
+ * @param consistencyLevel Consistency level of query
* @param fields Field name/value to search for
* @return UniqueValueSet containing fields from the collection that exist in cassandra
* @throws ConnectionException on error connecting to Cassandra
[39/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Conflicts:
stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9d00c223
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9d00c223
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9d00c223
Branch: refs/heads/USERGRID-405
Commit: 9d00c2235111954abbbcf9e26afacca74df1c1bd
Parents: a1b557d 361060e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:23:53 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:23:53 2015 -0600
----------------------------------------------------------------------
.../main/groovy/configure_elasticsearch.groovy | 1 -
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 31 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
12 files changed, 232 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
[35/50] incubator-usergrid git commit: add replay strategy
Posted by to...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44148384
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44148384
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44148384
Branch: refs/heads/USERGRID-405
Commit: 441483842b14c70bc075437a32cbef52f9460a3c
Parents: a1b557d
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 11:29:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 11:29:27 2015 -0600
----------------------------------------------------------------------
.../persistence/core/astyanax/ColumnNameIteratorTest.java | 4 ++--
.../core/astyanax/MultiKeyColumnNameIteratorTest.java | 4 ++--
.../persistence/core/astyanax/MultiRowColumnIteratorTest.java | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
index 0b0cbab..d2c6f89 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -76,12 +76,12 @@ public class ColumnNameIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
index 964c04a..f4f6f9c 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -80,12 +80,12 @@ public class MultiKeyColumnNameIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
index 11d34a4..c32b820 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -83,12 +83,12 @@ public class MultiRowColumnIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
[08/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e70391f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e70391f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e70391f8
Branch: refs/heads/USERGRID-405
Commit: e70391f8823642f9b5de44e92f06051a74160637
Parents: 67dd8c0 89f558a
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:11:53 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:11:53 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 10 +
.../EntityCollectionManagerFactoryImpl.java | 8 +-
.../impl/EntityCollectionManagerImpl.java | 120 ++++++--
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 +++++++++++++++++--
5 files changed, 382 insertions(+), 72 deletions(-)
----------------------------------------------------------------------