You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/18 21:56:01 UTC
[01/50] incubator-usergrid git commit: Added buffer wiring to guice
and updated the tests.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-460 4964ab62a -> 1087ee67a
Added buffer wiring to guice and updated the tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cd0015d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cd0015d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cd0015d5
Branch: refs/heads/USERGRID-460
Commit: cd0015d5af1371528943e015022be21b2d8099f9
Parents: c5a4767
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 14:56:47 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 14:56:47 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 9 ++++++++
.../persistence/index/guice/IndexModule.java | 10 ++++++++-
.../persistence/index/impl/BufferQueue.java | 7 ++++++
.../index/impl/BufferQueueInMemory.java | 18 ++++++++-------
.../index/impl/EsIndexBufferConsumerImpl.java | 23 +++++++++++++++-----
.../index/impl/EsIndexBufferProducerImpl.java | 2 +-
.../index/guice/TestIndexModule.java | 9 +++++++-
7 files changed, 62 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index c6f08f6..befbaa9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -64,6 +64,11 @@ public interface IndexFig extends GuicyFig {
*/
public static final String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple backpressure
+ */
+ public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -158,4 +163,8 @@ public interface IndexFig extends GuicyFig {
@Default("one")
@Key( INDEX_WRITE_CONSISTENCY_LEVEL )
String getWriteConsistencyLevel();
+
+ @Default("1000")
+ @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
+ long getFailureRetryTime();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index ebd9098..d911dab 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -32,7 +32,7 @@ import org.apache.usergrid.persistence.map.guice.MapModule;
import org.safehaus.guicyfig.GuicyFigModule;
-public class IndexModule extends AbstractModule {
+public abstract class IndexModule extends AbstractModule {
@Override
protected void configure() {
@@ -48,6 +48,14 @@ public class IndexModule extends AbstractModule {
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
+ wireBufferQueue();
}
+
+ /**
+ * Write the <class>BufferQueue</class> for this implementation
+ */
+ public abstract void wireBufferQueue();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
index dec6ac3..ffc3b90 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
@@ -46,4 +46,11 @@ public interface BufferQueue {
* @return
*/
public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
+
+
+ /**
+ * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented
+ * @param messages
+ */
+ public void ack(List<IndexOperationMessage> messages);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
index 502f45d..403762f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
@@ -36,15 +36,11 @@ import com.google.inject.Singleton;
public class BufferQueueInMemory implements BufferQueue {
private final ArrayBlockingQueue<IndexOperationMessage> messages;
- private final IndexFig fig;
@Inject
- public BufferQueueInMemory( final ArrayBlockingQueue<IndexOperationMessage> messages, final IndexFig fig ) {
- this.messages = messages;
-
-
- this.fig = fig;
+ public BufferQueueInMemory(final IndexFig fig ) {
+ messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
}
@@ -78,8 +74,14 @@ public class BufferQueueInMemory implements BufferQueue {
}
}
- while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
+ while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
+
+ return response;
+ }
+
- return null;
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+ //no op for this
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 19b8438..45c12a1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -39,6 +39,7 @@ import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequestBuilder;
import org.elasticsearch.client.Client;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -70,9 +71,8 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final BufferQueue bufferQueue;
@Inject
- public EsIndexBufferConsumerImpl( final IndexFig config, final IndexBufferProducer producer, final EsProvider
- provider, final MetricsFactory metricsFactory,
- final BufferQueue bufferQueue ){
+ public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider
+ provider, final MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
this.bufferQueue = bufferQueue;
this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
@@ -101,17 +101,30 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
for ( IndexOperationMessage drained : drainList ) {
subscriber.onNext( drained );
}
- drainList.clear();
+
+ bufferQueue.ack( drainList );
+
timer.stop();
countFail.set( 0 );
}
+ catch( EsRejectedExecutionException err) {
+ countFail.incrementAndGet();
+ log.error( "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. Failed {} consecutive times", config.getFailRefreshCount(), countFail.get() );
+
+ //es rejected the exception, sleep and retry in the queue
+ try {
+ Thread.sleep( config.getFailureRetryTime() );
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+ }
catch ( Exception e ) {
int count = countFail.incrementAndGet();
log.error( "failed to dequeue", e );
if ( count > 200 ) {
log.error( "Shutting down index drain due to repetitive failures" );
- //break;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
index f9999b2..db1f50e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
@@ -46,7 +46,7 @@ public class EsIndexBufferProducerImpl implements IndexBufferProducer {
private final BufferQueue bufferQueue;
@Inject
- public EsIndexBufferProducerImpl( MetricsFactory metricsFactory, IndexFig fig, final BufferQueue bufferQueue ){
+ public EsIndexBufferProducerImpl( MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
this.bufferQueue = bufferQueue;
this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferProducerImpl.class, "index.buffer.size");
this.timer = metricsFactory.getTimer(EsIndexBufferProducerImpl.class,"index.buffer.producer.timer");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cd0015d5/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 4d68dda..7e2312d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.index.guice;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
+import org.apache.usergrid.persistence.index.impl.BufferQueueInMemory;
public class TestIndexModule extends TestModule {
@@ -32,6 +34,11 @@ public class TestIndexModule extends TestModule {
// configure collections and our core astyanax framework
install( new CollectionModule() );
- install( new IndexModule() );
+ install( new IndexModule() {
+ @Override
+ public void wireBufferQueue() {
+ bind( BufferQueue.class).to( BufferQueueInMemory.class );
+ }
+ } );
}
}
[27/50] incubator-usergrid git commit: Added the ability to specify
the number of workers
Posted by gr...@apache.org.
Added the ability to specify the number of workers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9aec790b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9aec790b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9aec790b
Branch: refs/heads/USERGRID-460
Commit: 9aec790b3fd8486ce6dba67125f94a5d912e8596
Parents: d81dfaa
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 18:27:16 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 18:27:16 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 9 ++++
.../index/impl/EsIndexBufferConsumerImpl.java | 55 +++++++++++++++-----
.../index/guice/TestIndexModule.java | 4 +-
3 files changed, 52 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index cde86fd..445789f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -81,6 +81,11 @@ public interface IndexFig extends GuicyFig {
*/
public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+ /**
+ * The number of worker threads to consume from the queue
+ */
+ public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -181,4 +186,8 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_QUEUE_READ_TIMEOUT )
int getIndexQueueTimeout();
+ @Default("2")
+ @Key( ELASTICSEARCH_WORKER_COUNT )
+ int getWorkerCount();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 862b1ae..836ec3d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -49,9 +49,12 @@ import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
/**
* Consumer for IndexOperationMessages
@@ -69,15 +72,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Meter flushMeter;
private final Timer produceTimer;
private final BufferQueue bufferQueue;
+ private final IndexFig indexFig;
+ private final AtomicLong counter = new AtomicLong( );
//the actively running subscription
- private Subscription subscription;
+ private List<Subscription> subscriptions;
private Object mutex = new Object();
@Inject
- public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider
- provider, final MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
+ public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
+ metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
+
this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
@@ -86,15 +92,42 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
this.client = provider.getClient();
this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
this.bufferQueue = bufferQueue;
+ this.indexFig = indexFig;
-
+ subscriptions = new ArrayList<>( indexFig.getWorkerCount() );
//batch up sets of some size and send them in batch
start();
}
+ /**
+ * Loop throught and start the workers
+ */
public void start() {
+ final int count = indexFig.getWorkerCount();
+
+ for(int i = 0; i < count; i ++){
+ startWorker();
+ }
+ }
+
+
+ /**
+ * Stop the workers
+ */
+ public void stop() {
+ synchronized ( mutex ) {
+ //stop consuming
+
+ for(final Subscription subscription: subscriptions){
+ subscription.unsubscribe();
+ }
+ }
+ }
+
+
+ private void startWorker(){
synchronized ( mutex) {
final AtomicInteger countFail = new AtomicInteger();
@@ -104,7 +137,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
//name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
+ Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
List<IndexOperationMessage> drainList;
do {
@@ -168,20 +201,14 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
} );
//start in the background
- subscription = consumer.subscribe();
- }
- }
+ final Subscription subscription = consumer.subscribe();
- public void stop() {
- synchronized ( mutex ) {
- //stop consuming
- if(subscription != null) {
- subscription.unsubscribe();
- }
+ subscriptions.add(subscription );
}
}
+
/**
* Execute the request, check for errors, then re-init the batch for future use
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aec790b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7d7a18d..57c2fab 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -38,8 +38,8 @@ public class TestIndexModule extends TestModule {
install( new IndexModule() {
@Override
public void wireBufferQueue() {
-// bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
- bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
+ bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
+// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
}
[12/50] incubator-usergrid git commit: Fixes transitive dependency
issue with queue module
Posted by gr...@apache.org.
Fixes transitive dependency issue with queue module
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/67dd8c02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/67dd8c02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/67dd8c02
Branch: refs/heads/USERGRID-460
Commit: 67dd8c02b7949d6ed212ce0d41e83baa6a74e797
Parents: e58597b
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:11:49 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:11:49 2015 -0600
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/corepersistence/CoreModule.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67dd8c02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7b53d67..82beb8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -78,7 +78,7 @@ public class CoreModule extends AbstractModule {
}
} );
install(new MapModule());
- install(new QueueModule());
+// install(new QueueModule()); TODO, re-enable when index module doesn't depend on queue
bind(ManagerCache.class).to( CpManagerCache.class );
[48/50] incubator-usergrid git commit: Added more descriptive error
message
Posted by gr...@apache.org.
Added more descriptive error message
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3a47cf37
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3a47cf37
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3a47cf37
Branch: refs/heads/USERGRID-460
Commit: 3a47cf376b5c88734ab7111b2b504bab9e4cd5c5
Parents: e6fb121
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 11:05:42 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 11:05:42 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a47cf37/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cecd297..1838eec 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -293,7 +293,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final boolean isAcknowledged = result.isAcknowledged();
if(!isAcknowledged){
- throw new RuntimeException( "Unable to add aliases to the new index " + indexSuffix );
+ throw new RuntimeException( "Unable to add aliases to the new index. Elasticsearch did not acknowledge to the alias change for index '" + indexSuffix + "'");
}
}
[06/50] incubator-usergrid git commit: SQS buffer tests pass.
Posted by gr...@apache.org.
SQS buffer tests pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b3e42ddb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b3e42ddb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b3e42ddb
Branch: refs/heads/USERGRID-460
Commit: b3e42ddb3ca4182f675a650afe56688b32472e81
Parents: 8d8eb06
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 22:31:55 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 22:31:55 2015 -0600
----------------------------------------------------------------------
.../index/impl/BufferQueueSQSImplTest.java | 53 ++++--
.../queue/impl/SQSQueueManagerImpl.java | 176 ++++++++++---------
2 files changed, 130 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3e42ddb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index 4a4672e..6922c15 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -26,26 +26,27 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
import net.jcip.annotations.NotThreadSafe;
-import static org.junit.Assert.*;
-
-
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
@RunWith(EsRunner.class)
@@ -59,21 +60,23 @@ public class BufferQueueSQSImplTest {
public MigrationManagerRule migrationManagerRule;
@Inject
- private BufferQueueSQSImpl bufferQueueSQS;
+ public QueueManagerFactory queueManagerFactory;
+
+ @Inject
+ public IndexFig indexFig;
@Inject
- private EsIndexBufferConsumerImpl esIndexBufferConsumer;
+ public MapManagerFactory mapManagerFactory;
+ @Inject
+ public MetricsFactory metricsFactory;
- @Before
- public void stop() {
- esIndexBufferConsumer.stop();
- }
+ private BufferQueueSQSImpl bufferQueueSQS;
- @After
- public void after() {
- esIndexBufferConsumer.start();
+ @Before
+ public void setup(){
+ bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
}
@@ -82,6 +85,10 @@ public class BufferQueueSQSImplTest {
@Test
public void testMessageIndexing(){
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+ assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
+ assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
+
final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testType1", "testDoc1",request1Data );
@@ -112,9 +119,9 @@ public class BufferQueueSQSImplTest {
//now get it back
- final List<IndexOperationMessage> ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+ final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
- assertTrue(ops.size() > 1);
+ assertTrue(ops.size() > 0);
final IndexOperationMessage returnedOperation = ops.get( 0 );
@@ -139,6 +146,18 @@ public class BufferQueueSQSImplTest {
}
+ private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
+ final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+
+ List<IndexOperationMessage> ops;
+
+ do{
+ ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+ }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
+
+ return ops;
+ }
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3e42ddb/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index e2c5c1e..a78fc80 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -17,85 +17,106 @@
*/
package org.apache.usergrid.persistence.queue.impl;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SDKGlobalConfiguration;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.queue.Queue;
+import org.apache.usergrid.persistence.queue.QueueFig;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+
+import com.amazonaws.AbortedException;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
-import com.amazonaws.services.sqs.model.*;
+import com.amazonaws.services.sqs.model.BatchResultErrorEntry;
+import com.amazonaws.services.sqs.model.CreateQueueRequest;
+import com.amazonaws.services.sqs.model.CreateQueueResult;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchResult;
+import com.amazonaws.services.sqs.model.DeleteMessageRequest;
+import com.amazonaws.services.sqs.model.GetQueueUrlResult;
+import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.MessageAttributeValue;
+import com.amazonaws.services.sqs.model.QueueDoesNotExistException;
+import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
+import com.amazonaws.services.sqs.model.ReceiveMessageResult;
+import com.amazonaws.services.sqs.model.SendMessageBatchRequest;
+import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.inject.assistedinject.Assisted;
-import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
public class SQSQueueManagerImpl implements QueueManager {
private static final Logger LOG = LoggerFactory.getLogger(SQSQueueManagerImpl.class);
- private final AmazonSQSClient sqs;
+
private final QueueScope scope;
private ObjectMapper mapper;
+ private final QueueFig fig;
+ private final AmazonSQSClient sqs;
+
private static SmileFactory smileFactory = new SmileFactory();
- private LoadingCache<SqsLoader, Queue> urlMap = CacheBuilder.newBuilder()
- .maximumSize(1000)
- .build(new CacheLoader<SqsLoader, Queue>() {
- @Override
- public Queue load(SqsLoader queueLoader) throws Exception {
- Queue queue = null;
- try {
- GetQueueUrlResult result = sqs.getQueueUrl(queueLoader.getKey());
- queue = new Queue(result.getQueueUrl());
- } catch (QueueDoesNotExistException queueDoesNotExistException) {
- queue = null;
- } catch (Exception e) {
- LOG.error("failed to get queue from service", e);
- throw e;
- }
- if (queue == null) {
- String name = queueLoader.getKey();
- CreateQueueRequest createQueueRequest = new CreateQueueRequest()
- .withQueueName(name);
- CreateQueueResult result = sqs.createQueue(createQueueRequest);
- String url = result.getQueueUrl();
- queue = new Queue(url);
- LOG.info("Created queue with url {}", url);
- }
- return queue;
- }
- }
- );
+ private LoadingCache<String, Queue> urlMap = CacheBuilder.newBuilder()
+ .maximumSize( 1000 )
+ .build( new CacheLoader<String, Queue>() {
+ @Override
+ public Queue load( String queueLoader ) throws Exception {
+
+ //the amazon client is not thread safe, we need to create one per queue
+ Queue queue = null;
+ try {
+ GetQueueUrlResult result = sqs.getQueueUrl( queueLoader );
+ queue = new Queue( result.getQueueUrl() );
+ }catch ( QueueDoesNotExistException queueDoesNotExistException ) {
+ //no op, swallow
+
+ }
+ catch ( Exception e ) {
+ LOG.error( "failed to get queue from service", e );
+ throw e;
+ }
+ if ( queue == null ) {
+ CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueLoader );
+ CreateQueueResult result = sqs.createQueue( createQueueRequest );
+ String url = result.getQueueUrl();
+ queue = new Queue( url );
+ LOG.info( "Created queue with url {}", url );
+ }
+ return queue;
+ }
+ } );
+
@Inject
- public SQSQueueManagerImpl(@Assisted QueueScope scope, QueueFig fig){
+ public SQSQueueManagerImpl( @Assisted QueueScope scope, QueueFig fig ){
this.scope = scope;
+ this.fig = fig;
try {
- UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- this.sqs = new AmazonSQSClient(ugProvider.getCredentials());
- Regions regions = Regions.fromName(fig.getRegion());
- Region region = Region.getRegion(regions);
- sqs.setRegion(region);
+
smileFactory.delegateToTextual(true);
mapper = new ObjectMapper( smileFactory );
//pretty print, disabling for speed
// mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
+
+ sqs = createClient();
+
} catch ( Exception e ) {
throw new RuntimeException("Error setting up mapper", e);
}
@@ -109,7 +130,7 @@ public class SQSQueueManagerImpl implements QueueManager {
public Queue getQueue() {
try {
- Queue queue = urlMap.get(new SqsLoader(getName(),sqs));
+ Queue queue = urlMap.get(getName());
return queue;
} catch (ExecutionException ee) {
throw new RuntimeException(ee);
@@ -228,38 +249,29 @@ public class SQSQueueManagerImpl implements QueueManager {
return mapper.writeValueAsString(o);
}
- public class SqsLoader {
- private final String key;
-
- public SqsLoader(String key, AmazonSQSClient client) {
- this.key = key;
- }
-
- public String getKey() {
- return key;
- }
+ /**
+ * Get the region
+ * @return
+ */
+ private Region getRegion() {
+ Regions regions = Regions.fromName( fig.getRegion() );
+ Region region = Region.getRegion( regions );
+ return region;
+ }
- @Override
- public boolean equals(Object o){
- if(o instanceof SqsLoader){
- SqsLoader loader = (SqsLoader)o;
- return loader.getKey().equals(this.getKey());
- }
- return false;
- }
- @Override
- public int hashCode() {
- int result = getKey().hashCode();
- return result;
- }
+ /**
+ * Create the SQS client for the specified settings
+ */
+ private AmazonSQSClient createClient() {
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+ final AmazonSQSClient sqs = new AmazonSQSClient( ugProvider.getCredentials() );
+ final Region region = getRegion();
+ sqs.setRegion( region );
+ return sqs;
+ }
- @Override
- public String toString() {
- return getKey();
- }
- }
}
[35/50] incubator-usergrid git commit: add replay strategy
Posted by gr...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44148384
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44148384
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44148384
Branch: refs/heads/USERGRID-460
Commit: 441483842b14c70bc075437a32cbef52f9460a3c
Parents: a1b557d
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 11:29:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 11:29:27 2015 -0600
----------------------------------------------------------------------
.../persistence/core/astyanax/ColumnNameIteratorTest.java | 4 ++--
.../core/astyanax/MultiKeyColumnNameIteratorTest.java | 4 ++--
.../persistence/core/astyanax/MultiRowColumnIteratorTest.java | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
index 0b0cbab..d2c6f89 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -76,12 +76,12 @@ public class ColumnNameIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
index 964c04a..f4f6f9c 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -80,12 +80,12 @@ public class MultiKeyColumnNameIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44148384/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
index 11d34a4..c32b820 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -83,12 +83,12 @@ public class MultiRowColumnIteratorTest {
final CassandraConfig cassandraConfig = new CassandraConfig() {
@Override
public ConsistencyLevel getReadCL() {
- return ConsistencyLevel.CL_QUORUM;
+ return ConsistencyLevel.CL_LOCAL_ONE;
}
@Override
public ConsistencyLevel getConsistentReadCL() {
- return ConsistencyLevel.CL_LOCAL_ONE;
+ return ConsistencyLevel.CL_LOCAL_QUORUM;
}
[30/50] incubator-usergrid git commit: Fixes issue with duplicate
pool size definition
Posted by gr...@apache.org.
Fixes issue with duplicate pool size definition
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/97634dfc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/97634dfc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/97634dfc
Branch: refs/heads/USERGRID-460
Commit: 97634dfc6c5a8e7192d72567d68fd62bfd67a012
Parents: 19c6ad0
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 12 12:14:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 12 12:14:58 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_elasticsearch.groovy | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/97634dfc/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 38b032f..173e4e6 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -87,7 +87,6 @@ threadpool:
bulk:
type: fixed
size: 160
- size: 16
queue_size: 1000
search:
size: 320
[15/50] incubator-usergrid git commit: Fixes issue with Guice
transitive dependency
Posted by gr...@apache.org.
Fixes issue with Guice transitive dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/67fa78ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/67fa78ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/67fa78ef
Branch: refs/heads/USERGRID-460
Commit: 67fa78efd6df6bd365972c7ed40a1d16dd8caac6
Parents: e70391f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:45:15 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:45:15 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/CoreModule.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67fa78ef/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 82beb8d..3230faa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -77,8 +77,8 @@ public class CoreModule extends AbstractModule {
bind(BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
- install(new MapModule());
-// install(new QueueModule()); TODO, re-enable when index module doesn't depend on queue
+// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
+// install(new QueueModule());
bind(ManagerCache.class).to( CpManagerCache.class );
[37/50] incubator-usergrid git commit: add comment
Posted by gr...@apache.org.
add comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8cea08db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8cea08db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8cea08db
Branch: refs/heads/USERGRID-460
Commit: 8cea08db37eafede2dbe3c7a8cd1fd38dc0d548a
Parents: 117c0f3
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:21:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:21:27 2015 -0600
----------------------------------------------------------------------
.../collection/serialization/UniqueValueSerializationStrategy.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8cea08db/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 8d314d8..60275d9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -64,6 +64,7 @@ public interface UniqueValueSerializationStrategy extends Migration {
* Load UniqueValue that matches field from collection or null if that value does not exist.
*
* @param colScope Collection scope in which to look for field name/value
+ * @param consistencyLevel Consistency level of query
* @param fields Field name/value to search for
* @return UniqueValueSet containing fields from the collection that exist in cassandra
* @throws ConnectionException on error connecting to Cassandra
[09/50] incubator-usergrid git commit: add timer and meter to
collection
Posted by gr...@apache.org.
add timer and meter to collection
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8ccc3ec5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8ccc3ec5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8ccc3ec5
Branch: refs/heads/USERGRID-460
Commit: 8ccc3ec5cf9a29381f2acdb63331f9de447dfd64
Parents: 06fe609
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 09:04:41 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 09:04:41 2015 -0600
----------------------------------------------------------------------
.../EntityCollectionManagerFactoryImpl.java | 8 +-
.../impl/EntityCollectionManagerImpl.java | 120 ++++++++++++++-----
2 files changed, 99 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ccc3ec5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 4e04c2e..bb1b56a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import java.util.concurrent.ExecutionException;
@@ -80,6 +81,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final EntityDeletedFactory entityDeletedFactory;
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
+ private final MetricsFactory metricsFactory;
private LoadingCache<CollectionScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
@@ -91,7 +93,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
mvccLogEntrySerializationStrategy, keyspace, entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope );
+ entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -117,7 +119,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final EntityVersionCreatedFactory entityVersionCreatedFactory,
final EntityDeletedFactory entityDeletedFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
- final EntityCacheFig entityCacheFig) {
+ final EntityCacheFig entityCacheFig,
+ MetricsFactory metricsFactory) {
this.writeStart = writeStart;
this.writeUpdate = writeUpdate;
@@ -136,6 +139,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.entityDeletedFactory = entityDeletedFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
+ this.metricsFactory = metricsFactory;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ccc3ec5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 7c467c6..391e9a5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,6 +23,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,8 +75,10 @@ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
+import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -113,6 +118,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final TaskExecutor taskExecutor;
private final Keyspace keyspace;
+ private final Timer writeTimer;
+ private final Meter writeMeter;
+ private final Timer deleteTimer;
+ private final Timer updateTimer;
+ private final Timer loadTimer;
+ private final Timer getLatestTimer;
@Inject
@@ -133,7 +144,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final EntityVersionCreatedFactory entityVersionCreatedFactory,
final EntityDeletedFactory entityDeletedFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
+ @Assisted final CollectionScope collectionScope,
+ final MetricsFactory metricsFactory
+
) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
@@ -160,6 +173,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.collectionScope = collectionScope;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
+ this.writeTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"write.timer");
+ this.writeMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "write.meter");
+ this.deleteTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "delete.timer");
+ this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"update.timer");
+ this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"load.timer");
+ this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"latest.timer");
}
@@ -184,6 +203,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
+ final Timer.Context timer = writeTimer.time();
return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
public void call(final Entity entity) {
@@ -192,7 +212,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
//post-processing to come later. leave it empty for now.
}
- }).doOnError(rollback);
+ }).doOnError(rollback)
+ .doOnEach(new Action1<Notification<? super Entity>>() {
+ @Override
+ public void call(Notification<? super Entity> notification) {
+ writeMeter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -203,21 +235,27 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
+ final Timer.Context timer = deleteTimer.time();
Observable<Id> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId))
- .map( markStart)
- .doOnNext( markCommit)
- .map( new Func1<CollectionIoEvent<MvccEntity>, Id>() {
-
+ .map(markStart)
+ .doOnNext(markCommit)
+ .map(new Func1<CollectionIoEvent<MvccEntity>, Id>() {
+
+ @Override
+ public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
+ MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
+ Task<Void> task = entityDeletedFactory
+ .getTask(collectionScope, entity.getId(), entity.getVersion());
+ taskExecutor.submit(task);
+ return entity.getId();
+ }
+ }
+ ) .doOnCompleted(new Action0() {
@Override
- public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityDeletedFactory
- .getTask( collectionScope, entity.getId(), entity.getVersion());
- taskExecutor.submit(task);
- return entity.getId();
+ public void call() {
+ timer.stop();
}
- }
- );
+ });;
return o;
}
@@ -230,18 +268,25 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return load( Collections.singleton( entityId ) ).flatMap( new Func1<EntitySet, Observable<Entity>>() {
+ final Timer.Context timer = loadTimer.time();
+ return load( Collections.singleton( entityId ) ).flatMap(new Func1<EntitySet, Observable<Entity>>() {
@Override
- public Observable<Entity> call( final EntitySet entitySet ) {
- final MvccEntity entity = entitySet.getEntity( entityId );
+ public Observable<Entity> call(final EntitySet entitySet) {
+ final MvccEntity entity = entitySet.getEntity(entityId);
- if ( entity == null || !entity.getEntity().isPresent() ) {
+ if (entity == null || !entity.getEntity().isPresent()) {
return Observable.empty();
}
- return Observable.from( entity.getEntity().get() );
+ return Observable.from(entity.getEntity().get());
}
- } );
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -253,6 +298,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
+ final Timer.Context timer = loadTimer.time();
return Observable.create( new Observable.OnSubscribe<EntitySet>() {
@@ -269,7 +315,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } );
+ } )
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -314,13 +366,14 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
- return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
+ final Timer.Context timer = updateTimer.time();
+ return observable.map( writeCommit ).doOnNext(new Action1<Entity>() {
@Override
- public void call( final Entity entity ) {
- logger.debug( "sending entity to the queue" );
+ public void call(final Entity entity) {
+ logger.debug("sending entity to the queue");
//we an update, signal the fix
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
+ taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope, entity));
//TODO T.N Change this to fire a task
// Observable.from( new CollectionIoEvent<Id>(collectionScope,
@@ -328,7 +381,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
- } ).doOnError( rollback );
+ }).doOnError(rollback)
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@@ -362,6 +421,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
+ final Timer.Context timer = getLatestTimer.time();
return Observable.create( new Observable.OnSubscribe<VersionSet>() {
@Override
@@ -377,7 +437,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } );
+ } )
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
[36/50] incubator-usergrid git commit: add replay strategy
Posted by gr...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/117c0f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/117c0f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/117c0f3d
Branch: refs/heads/USERGRID-460
Commit: 117c0f3d8839cb1da2cae35b9a1171c1b404ac98
Parents: 4414838
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:10:55 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:10:55 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 94 +++++++++-----------
1 file changed, 42 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/117c0f3d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 5f96b92..173efa7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -101,9 +101,46 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final CollectionScope scope = ioevent.getEntityCollection();
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+ //allocate our max size, worst case
+ final List<Field> uniqueFields = new ArrayList<>( entity.getFields().size() );
+
+ //
+ // Construct all the functions for verifying we're unique
+ //
+ for ( final Field field : entity.getFields() ) {
+
+ // if it's unique, create a function to validate it and add it to the list of
+ // concurrent validations
+ if ( field.isUnique() ) {
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
+
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+
+ batch.mergeShallow( mb );
+ uniqueFields.add(field);
+ }
+ }
+
+ //short circuit nothing to do
+ if ( uniqueFields.size() == 0 ) {
+ return ;
+ }
+
+ //perform the write
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException ex ) {
+ throw new RuntimeException( "Unable to write to cassandra", ex );
+ }
+
// use simple thread pool to verify fields in parallel
- ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,keyspace,serializationFig,cassandraFig,scope,entity);
+ ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,cassandraFig,scope, uniqueFields,entity);
Map<String,Field> uniquenessViolations = cmd.execute();
+ cmd.getFailedExecutionException();
//We have violations, throw an exception
if ( !uniquenessViolations.isEmpty() ) {
throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
@@ -113,19 +150,17 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
private static class ConsistentReplayCommand extends HystrixCommand<Map<String,Field>>{
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final Keyspace keySpace;
- private final SerializationFig serializationFig;
private final CassandraConfig fig;
private final CollectionScope scope;
+ private final List<Field> uniqueFields;
private final Entity entity;
- public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy,Keyspace keySpace, SerializationFig serializationFig, CassandraConfig fig,CollectionScope scope, Entity entity){
+ public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy, CassandraConfig fig, CollectionScope scope, List<Field> uniqueFields, Entity entity){
super(REPLAY_GROUP);
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
- this.keySpace = keySpace;
- this.serializationFig = serializationFig;
this.fig = fig;
this.scope = scope;
+ this.uniqueFields = uniqueFields;
this.entity = entity;
}
@@ -142,23 +177,17 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
Collection<Field> entityFields = entity.getFields();
//allocate our max size, worst case
- final List<Field> uniqueFields = new ArrayList<Field>( entityFields.size() );
//now get the set of fields back
final UniqueValueSet uniqueValues;
- //todo add consistencylevel and read back if fail using
-
try {
-
- uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, entityFields );
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
}
-
final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
-
//loop through each field that was unique
for ( final Field field : uniqueFields ) {
@@ -170,51 +199,12 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
field.getName() ) );
}
-
final Id returnedEntityId = uniqueValue.getEntityId();
-
if ( !entity.getId().equals(returnedEntityId) ) {
uniquenessViolations.put( field.getName(), field );
}
}
- final MutationBatch batch = keySpace.prepareMutationBatch();
- //
- // Construct all the functions for verifying we're unique
- //
- for ( final Field field : entity.getFields() ) {
-
- // if it's unique, create a function to validate it and add it to the list of
- // concurrent validations
- if ( field.isUnique() ) {
-
-
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, entity.getId(), entity.getVersion() );
-
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueSerializationStrategy.write( scope, written, serializationFig.getTimeout() );
-
- batch.mergeShallow( mb );
-
-
- uniqueFields.add(field);
- }
- }
-
- //short circuit nothing to do
- if ( uniqueFields.size() == 0 ) {
- return uniquenessViolations ;
- }
-
-
- //perform the write
- try {
- batch.execute();
- }
- catch ( ConnectionException ex ) {
- throw new RuntimeException( "Unable to write to cassandra", ex );
- }
return uniquenessViolations;
}
[44/50] incubator-usergrid git commit: merge from two-o
Posted by gr...@apache.org.
merge from two-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b53cb07f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b53cb07f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b53cb07f
Branch: refs/heads/USERGRID-460
Commit: b53cb07f376fcdff8ce0b3226bbb07843f12d4e9
Parents: 0334119 b0a07ba
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 16 17:35:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 16 17:35:24 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 49 ++++++++++-
.../usergrid/persistence/EntityManager.java | 10 ++-
.../cassandra/EntityManagerImpl.java | 8 +-
.../usergrid/persistence/EntityManagerIT.java | 1 +
.../collection/EntityCollectionManager.java | 13 ++-
.../persistence/collection/FieldSet.java | 48 ++++++++++
.../cache/CachedEntityCollectionManager.java | 11 ++-
.../impl/EntityCollectionManagerImpl.java | 92 +++++++++++++++++++-
.../serialization/impl/MutableFieldSet.java | 63 ++++++++++++++
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 72 ++++++++++++++-
.../services/AbstractCollectionService.java | 40 +++------
.../services/AbstractConnectionsService.java | 39 +++------
13 files changed, 374 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e2b7bd1,e5223f0..789e640
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -33,7 -33,7 +33,8 @@@ import java.util.TreeMap
import java.util.TreeSet;
import java.util.UUID;
+import com.codahale.metrics.Meter;
+ import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -206,8 -207,7 +207,9 @@@ public class CpEntityManager implement
private Timer entGetEntityCountersTimer;
private Timer esIndexEntityCollectionTimer;
private Timer entRevokeRolePermissionsTimer;
+ private Timer entGetRepairedEntityTimer;
+ private Timer updateEntityTimer;
+ private Meter updateEntityMeter;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@@ -268,10 -268,9 +270,12 @@@
.getTimer( CpEntityManager.class, "cp.entity.es.index.entity.to.collection.timer" );
this.entRevokeRolePermissionsTimer =
this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+ this.entGetRepairedEntityTimer = this.metricsFactory
+ .getTimer( CpEntityManager.class, "get.repaired.entity.timer" );
+ this.updateEntityMeter =this.metricsFactory.getMeter(CpEntityManager.class,"cp.entity.update.meter");
+ this.updateEntityTimer =this.metricsFactory.getTimer(CpEntityManager.class, "cp.entity.update.timer");
+
// set to false for now
this.skipAggregateCounters = false;
@@@ -566,12 -577,8 +581,11 @@@
Preconditions.checkNotNull(appId,"app scope should never be null");
// first, update entity index in its own collection scope
+ updateEntityMeter.mark();
+ Timer.Context timer = updateEntityTimer.time();
+
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
-
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
if ( logger.isDebugEnabled() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 92b07f0,3c89280..6ba4513
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -70,11 -61,12 +72,14 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
+ import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.core.task.Task;
+ import org.apache.usergrid.persistence.core.task.TaskExecutor;
+import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action0;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b53cb07f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
[22/50] incubator-usergrid git commit: Fixed naming convention issue
Posted by gr...@apache.org.
Fixed naming convention issue
Fixes blocking on cleanup (not necessary)
Fixes index creation when alias is cached
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c5943465
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c5943465
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c5943465
Branch: refs/heads/USERGRID-460
Commit: c594346558b9624c05f62096f8d83a0a91ce8af5
Parents: 9630fcf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:02:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:02:58 2015 -0600
----------------------------------------------------------------------
.../results/FilteringLoader.java | 2 +-
.../index/impl/BufferQueueSQSImpl.java | 14 +-
.../index/impl/EsEntityIndexBatchImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 14 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 19 +--
.../persistence/index/impl/EsIndexCache.java | 138 +++++++++++--------
.../queue/impl/SQSQueueManagerImpl.java | 10 +-
7 files changed, 124 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 1afd76b..2cd9fdc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -233,7 +233,7 @@ public class FilteringLoader implements ResultsLoader {
@Override
public void postProcess() {
- this.indexBatch.execute().get();
+ this.indexBatch.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 25c2ba6..3cace11 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -127,6 +127,12 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
+ //no op
+ if(operation.isEmpty()){
+ operation.getFuture().done();
+ return;
+ }
+
final Timer.Context timer = this.writeTimer.time();
this.writeMeter.mark();
@@ -141,7 +147,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//signal to SQS
this.queue.sendMessage( identifier );
- operation.getFuture().run();
+ operation.getFuture().done();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -184,7 +190,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
//look up the values
- final Map<String, String> values = mapManager.getStrings( mapEntries );
+ final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
//load them into our response
@@ -193,7 +199,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final String key = getMessageKey( message );
//now see if the key was there
- final String payload = values.get( key );
+ final String payload = storedCommands.get( key );
//the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
// a DLQ
@@ -242,7 +248,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
- final String key = getMessageKey(sqsIndexOperationMessage.getMessage());
+ final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
//remove it from the map
mapManager.delete( key );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b63dfe6..8481dab 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -197,7 +197,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* No-op, just disregard it
*/
if(tempContainer.isEmpty()){
- return tempContainer.getFuture();
+ final BetterFuture<?> future = tempContainer.getFuture();
+ future.done();
+ return future;
}
return indexBatchBufferProducer.put(tempContainer);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 99643da..fa50734 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -209,7 +209,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- String[] indexes = getIndexes(AliasType.Write);
+ String[] indexes = getIndexesFromEs(AliasType.Write);
if(indexes == null || indexes.length==0) {
addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
}
@@ -336,6 +336,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
/**
+ * Get our index info from ES, but clear our cache first
+ * @param aliasType
+ * @return
+ */
+ public String[] getIndexesFromEs(final AliasType aliasType){
+ aliasCache.invalidate( alias );
+ return getIndexes( aliasType );
+ }
+
+
+
+ /**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
* http://s.apache.org/index-missing-exception
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 2762c18..862b1ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -46,6 +46,7 @@ import rx.Subscription;
import rx.functions.Action1;
import rx.functions.Action2;
import rx.functions.Func1;
+import rx.functions.Func2;
import rx.schedulers.Schedulers;
import java.util.List;
@@ -191,27 +192,27 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
//process and flatten all the messages to builder requests
- Observable<IndexOperationMessage> flattenMessages = Observable.from( operationMessages );
-
-
//batch shard operations into a bulk request
- flattenMessages.flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
+ Observable.from( operationMessages ).flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
@Override
public Observable<BatchRequest> call( final IndexOperationMessage indexOperationMessage ) {
final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
- final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ final Observable<DeIndexRequest> deIndex =
+ Observable.from( indexOperationMessage.getDeIndexRequests() );
- indexSizeCounter.inc(indexOperationMessage.getDeIndexRequests().size());
- indexSizeCounter.inc(indexOperationMessage.getIndexRequests().size());
+ indexSizeCounter.dec( indexOperationMessage.getDeIndexRequests().size() );
+ indexSizeCounter.dec( indexOperationMessage.getIndexRequests().size() );
return Observable.merge( index, deIndex );
}
} )
//collection all the operations into a single stream
- .collect( initRequest(), new Action2<BulkRequestBuilder, BatchRequest>() {
+ .reduce( initRequest(), new Func2<BulkRequestBuilder, BatchRequest, BulkRequestBuilder>() {
@Override
- public void call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
batchRequest.doOperation( client, bulkRequestBuilder );
+
+ return bulkRequestBuilder;
}
} )
//send the request off to ES
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index 0c07a34..ef518dd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -20,6 +20,24 @@
package org.apache.usergrid.persistence.index.impl;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -29,21 +47,6 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
/**
@@ -52,65 +55,88 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class EsIndexCache {
- private static final Logger logger = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<String, String[]> aliasIndexCache;
+ private EsProvider provider;
+
@Inject
- public EsIndexCache(final EsProvider provider, final IndexFig indexFig) {
-
- this.refreshExecutors = MoreExecutors
- .listeningDecorator(Executors.newScheduledThreadPool(indexFig.getIndexCacheMaxWorkers()));
-
- aliasIndexCache = CacheBuilder.newBuilder().maximumSize(1000)
- .refreshAfterWrite(5,TimeUnit.MINUTES)
- .build(new CacheLoader<String, String[]>() {
- @Override
- public ListenableFuture<String[]> reload(final String key, String[] oldValue) throws Exception {
- ListenableFutureTask<String[]> task = ListenableFutureTask.create( new Callable<String[]>() {
- public String[] call() {
- return load( key );
- }
- } );
- refreshExecutors.execute(task);
- return task;
- }
-
- @Override
- public String[] load(final String aliasName) {
- final AdminClient adminClient = provider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
- adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
- return aliasMap.keys().toArray(String.class);
- }
- }) ;
+ public EsIndexCache( final EsProvider provider, final IndexFig indexFig ) {
+
+ this.refreshExecutors =
+ MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
+
+ this.provider = provider;
+
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
+ .build( new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload( final String key,
+ String[] oldValue )
+ throws Exception {
+ ListenableFutureTask<String[]> task =
+ ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute( task );
+ return task;
+ }
+
+
+ @Override
+ public String[] load( final String aliasName ) {
+ return getIndexesFromEs(aliasName);
+ }
+ } );
}
-
+
/**
* Get indexes for an alias
*/
- public String[] getIndexes(IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
String[] indexes;
try {
- indexes = aliasIndexCache.get(aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias());
- } catch (ExecutionException ee) {
- logger.error("Failed to retreive indexes", ee);
- throw new RuntimeException(ee);
+ indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
+ }
+ catch ( ExecutionException ee ) {
+ logger.error( "Failed to retreive indexes", ee );
+ throw new RuntimeException( ee );
}
return indexes;
}
-
+
+
+ private String[] getIndexesFromEs(final String aliasName){
+ final AdminClient adminClient = this.provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
+ adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
+ return aliasMap.keys().toArray( String.class );
+ }
+
+
/**
- * clean up cache
+ * Get the name of the alias to use
+ * @param alias
+ * @param aliasType
+ * @return
*/
- public void invalidate(IndexIdentifier.IndexAlias alias){
- aliasIndexCache.invalidate(alias.getWriteAlias());
- aliasIndexCache.invalidate(alias.getReadAlias());
-
+ private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
}
+
+ /**
+ * clean up cache
+ */
+ public void invalidate( IndexIdentifier.IndexAlias alias ) {
+ aliasIndexCache.invalidate( alias.getWriteAlias() );
+ aliasIndexCache.invalidate( alias.getReadAlias() );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index a78fc80..1fbd9b6 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -33,7 +33,6 @@ import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueMessage;
import org.apache.usergrid.persistence.queue.QueueScope;
-import com.amazonaws.AbortedException;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
@@ -76,15 +75,16 @@ public class SQSQueueManagerImpl implements QueueManager {
.maximumSize( 1000 )
.build( new CacheLoader<String, Queue>() {
@Override
- public Queue load( String queueLoader ) throws Exception {
+ public Queue load( String queueName ) throws Exception {
//the amazon client is not thread safe, we need to create one per queue
Queue queue = null;
try {
- GetQueueUrlResult result = sqs.getQueueUrl( queueLoader );
+ GetQueueUrlResult result = sqs.getQueueUrl( queueName );
queue = new Queue( result.getQueueUrl() );
}catch ( QueueDoesNotExistException queueDoesNotExistException ) {
//no op, swallow
+ LOG.error( "Queue {} does not exist, creating", queueName );
}
catch ( Exception e ) {
@@ -92,7 +92,7 @@ public class SQSQueueManagerImpl implements QueueManager {
throw e;
}
if ( queue == null ) {
- CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueLoader );
+ CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueName );
CreateQueueResult result = sqs.createQueue( createQueueRequest );
String url = result.getQueueUrl();
queue = new Queue( url );
@@ -124,7 +124,7 @@ public class SQSQueueManagerImpl implements QueueManager {
private String getName() {
- String name = scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
+ String name = fig.getPrefix() + "_" + scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
return name;
}
[19/50] incubator-usergrid git commit: more meters
Posted by gr...@apache.org.
more meters
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/615e3fc9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/615e3fc9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/615e3fc9
Branch: refs/heads/USERGRID-460
Commit: 615e3fc9073530dcfef5774d53b744fe242ba5ea
Parents: 8fa5c9f
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 15:23:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 15:23:16 2015 -0600
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 39 ++++++++++++++++++--
.../graph/impl/GraphManagerImpl.java | 2 +-
2 files changed, 37 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615e3fc9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 391e9a5..50b581a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -124,6 +124,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final Timer updateTimer;
private final Timer loadTimer;
private final Timer getLatestTimer;
+ private final Meter deleteMeter;
+ private final Meter getLatestMeter;
+ private final Meter loadMeter;
+ private final Meter updateMeter;
@Inject
@@ -176,9 +180,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.writeTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"write.timer");
this.writeMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "write.meter");
this.deleteTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "delete.timer");
- this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"update.timer");
+ this.deleteMeter= metricsFactory.getMeter(EntityCollectionManagerImpl.class, "delete.meter");
+ this.updateTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "update.timer");
+ this.updateMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "update.meter");
this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"load.timer");
+ this.loadMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "load.meter");
this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class,"latest.timer");
+ this.getLatestMeter = metricsFactory.getMeter(EntityCollectionManagerImpl.class, "latest.meter");
}
@@ -250,12 +258,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return entity.getId();
}
}
- ) .doOnCompleted(new Action0() {
+ )
+ .doOnNext(new Action1<Id>() {
+ @Override
+ public void call(Id id) {
+ deleteMeter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
@Override
public void call() {
timer.stop();
}
- });;
+ });
return o;
}
@@ -281,6 +296,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return Observable.from(entity.getEntity().get());
}
})
+ .doOnNext(new Action1<Entity>() {
+ @Override
+ public void call(Entity entity) {
+ loadMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
@@ -316,6 +337,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}
} )
+ .doOnNext(new Action1<EntitySet>() {
+ @Override
+ public void call(EntitySet entitySet) {
+ loadMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
@@ -382,6 +409,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
}).doOnError(rollback)
+ .doOnNext(new Action1<Entity>() {
+ @Override
+ public void call(Entity entity) {
+ updateMeter.mark();
+ }
+ })
.doOnCompleted(new Action0() {
@Override
public void call() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615e3fc9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 53e116d..f19d613 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -166,7 +166,7 @@ public class GraphManagerImpl implements GraphManager {
this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.type.timer");
this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.from.timer");
- this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+ this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.from.meter");
this.getIdTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.idtype.from.timer");
this.getIdTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.from.meter");
[31/50] incubator-usergrid git commit: change consistency level
Posted by gr...@apache.org.
change consistency level
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9e6e36dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9e6e36dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9e6e36dd
Branch: refs/heads/USERGRID-460
Commit: 9e6e36ddb5bd1d38622b100b4916cac6dcbc1d77
Parents: e8dc17d
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 12 17:29:48 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 12 17:29:48 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 3 +-
.../UniqueValueSerializationStrategy.java | 20 ++++++++++----
.../UniqueValueSerializationStrategyImpl.java | 19 ++++++++-----
.../core/astyanax/CassandraConfig.java | 6 ++++
.../core/astyanax/CassandraConfigImpl.java | 8 ++++--
.../persistence/core/astyanax/CassandraFig.java | 9 ++++--
.../core/astyanax/ColumnNameIteratorTest.java | 5 ++++
.../MultiKeyColumnNameIteratorTest.java | 5 ++++
.../astyanax/MultiRowColumnIteratorTest.java | 5 ++++
.../index/impl/EsEntityIndexImpl.java | 29 +++++++++++---------
.../persistence/queue/QueueManagerTest.java | 2 +-
11 files changed, 80 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 162bcac..1c30e75 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,7 +111,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
//
for ( final Field field : entityFields ) {
- // if it's unique, create a function to validate it and add it to the list of
+ // if it's unique, create a function to validate it and add it to the list of
// concurrent validations
if ( field.isUnique() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 030d9d1..8d314d8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.ConsistencyLevel;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.field.Field;
@@ -34,7 +35,7 @@ public interface UniqueValueSerializationStrategy extends Migration {
/**
* Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds.
- *
+ *
* @param uniqueValue Object to be written
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
@@ -42,16 +43,16 @@ public interface UniqueValueSerializationStrategy extends Migration {
/**
* Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds.
- *
+ *
* @param uniqueValue Object to be written
- * @param timeToLive How long object should live in seconds
+ * @param timeToLive How long object should live in seconds
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
public MutationBatch write( CollectionScope scope, UniqueValue uniqueValue, Integer timeToLive );
/**
* Load UniqueValue that matches field from collection or null if that value does not exist.
- *
+ *
* @param colScope Collection scope in which to look for field name/value
* @param fields Field name/value to search for
* @return UniqueValueSet containing fields from the collection that exist in cassandra
@@ -60,8 +61,17 @@ public interface UniqueValueSerializationStrategy extends Migration {
public UniqueValueSet load( CollectionScope colScope, Collection<Field> fields ) throws ConnectionException;
/**
+ * Load UniqueValue that matches field from collection or null if that value does not exist.
+ *
+ * @param colScope Collection scope in which to look for field name/value
+ * @param fields Field name/value to search for
+ * @return UniqueValueSet containing fields from the collection that exist in cassandra
+ * @throws ConnectionException on error connecting to Cassandra
+ */
+ public UniqueValueSet load( CollectionScope colScope, ConsistencyLevel consistencyLevel, Collection<Field> fields ) throws ConnectionException;
+ /**
* Delete the specified Unique Value from Cassandra.
- *
+ *
* @param uniqueValue Object to be deleted.
* @return MutatationBatch that encapsulates operation, caller may or may not execute.
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index be95b08..6483408 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -25,6 +25,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import org.apache.usergrid.persistence.core.astyanax.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,10 +36,6 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
-import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
-import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -72,6 +70,7 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
ENTITY_VERSION_SER );
protected final Keyspace keyspace;
+ private final CassandraFig cassandraFig;
/**
@@ -80,7 +79,8 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
* @param keyspace Keyspace in which to store Unique Values.
*/
@Inject
- public UniqueValueSerializationStrategyImpl( final Keyspace keyspace ) {
+ public UniqueValueSerializationStrategyImpl( final Keyspace keyspace, final CassandraFig cassandraFig) {
+ this.cassandraFig = cassandraFig;
this.keyspace = keyspace;
}
@@ -171,8 +171,13 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
}
+
+ @Override
+ public UniqueValueSet load(final CollectionScope colScope, final Collection<Field> fields ) throws ConnectionException{
+ return load(colScope,ConsistencyLevel.valueOf(cassandraFig.getReadCL()), fields);
+ }
@Override
- public UniqueValueSet load(final CollectionScope colScope, final Collection<Field> fields )
+ public UniqueValueSet load(final CollectionScope colScope, final ConsistencyLevel consistencyLevel, final Collection<Field> fields )
throws ConnectionException {
Preconditions.checkNotNull( fields, "fields are required" );
@@ -198,7 +203,7 @@ public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializ
final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl( fields.size() );
Iterator<Row<ScopedRowKey<CollectionPrefixedKey<Field>>, EntityVersion>> results =
- keyspace.prepareQuery( CF_UNIQUE_VALUES ).getKeySlice( keys )
+ keyspace.prepareQuery( CF_UNIQUE_VALUES ).setConsistencyLevel(consistencyLevel).getKeySlice( keys )
.withColumnRange( new RangeBuilder().setLimit( 1 ).build() ).execute().getResult().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
index ad4463a..817aee2 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfig.java
@@ -37,6 +37,12 @@ public interface CassandraConfig {
public ConsistencyLevel getReadCL();
/**
+ * Get the currently configured ReadCL that is more consitent than getReadCL
+ * @return
+ */
+ public ConsistencyLevel getConsistentReadCL();
+
+ /**
* Get the currently configured write CL
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
index ce80ea2..17b91c6 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraConfigImpl.java
@@ -39,6 +39,7 @@ public class CassandraConfigImpl implements CassandraConfig {
private ConsistencyLevel readCl;
private ConsistencyLevel writeCl;
private int[] shardSettings;
+ private ConsistencyLevel consistentCl;
@Inject
@@ -50,7 +51,7 @@ public class CassandraConfigImpl implements CassandraConfig {
this.shardSettings = parseShardSettings( cassandraFig.getShardValues() );
-
+ this.consistentCl = ConsistencyLevel.valueOf(cassandraFig.getConsistentReadCL());
//add the listeners to update the values
cassandraFig.addPropertyChangeListener( new PropertyChangeListener() {
@@ -78,7 +79,10 @@ public class CassandraConfigImpl implements CassandraConfig {
return readCl;
}
-
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return consistentCl;
+ }
@Override
public ConsistencyLevel getWriteCL() {
return writeCl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index 1208a1a..66a716c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -33,6 +33,8 @@ import org.safehaus.guicyfig.Key;
public interface CassandraFig extends GuicyFig {
+ public static final String READ_CONSISTENT_CL = "usergrid.consistent.read.cl";
+
public static final String READ_CL = "usergrid.read.cl";
public static final String WRITE_CL = "usergrid.write.cl";
@@ -77,11 +79,14 @@ public interface CassandraFig extends GuicyFig {
@Default( "false" )
boolean isEmbedded();
-
- @Default("CL_QUORUM")
+ @Default("CL_ONE")
@Key(READ_CL)
String getReadCL();
+ @Default("CL_LOCAL_ONE")
+ @Key(READ_CONSISTENT_CL)
+ String getConsistentReadCL();
+
@Default("CL_QUORUM")
@Key(WRITE_CL)
String getWriteCL();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
index 547c504..0b0cbab 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -79,6 +79,11 @@ public class ColumnNameIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
index a96db39..964c04a 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -83,6 +83,11 @@ public class MultiKeyColumnNameIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
index 4530d4b..11d34a4 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -86,6 +86,11 @@ public class MultiRowColumnIteratorTest {
return ConsistencyLevel.CL_QUORUM;
}
+ @Override
+ public ConsistencyLevel getConsistentReadCL() {
+ return ConsistencyLevel.CL_LOCAL_ONE;
+ }
+
@Override
public ConsistencyLevel getWriteCL() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fa50734..9343d39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -113,6 +113,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer searchTimer;
private final Timer allVersionsTimerFuture;
private final Timer deletePreviousTimerFuture;
+ private final Meter errorMeter;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -174,31 +175,33 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.aliasCache = indexCache;
this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.timer" );
this.removeAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "remove.alias.timer" );
this.addReadAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.read.alias.timer" );
this.addWriteAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.write.alias.timer" );
this.mappingTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" );
+ .getTimer( EsEntityIndexImpl.class, "create.mapping.timer" );
this.refreshTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" );
+ .getTimer( EsEntityIndexImpl.class, "refresh.timer" );
this.searchTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer.future" );
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer.future" );
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
+
+ this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e6e36dd/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index 935fd16..e20f87c 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -63,7 +63,7 @@ public class QueueManagerTest {
@Before
public void mockApp() {
this.scope = new QueueScopeImpl( new SimpleId( "application" ), "testQueue" );
- qm = qmf.getQueueManager(scope);
+// qm = qmf.getQueueManager(scope);
queueScopeFactory = new QueueScopeFactoryImpl(queueFig);
}
[13/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e70391f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e70391f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e70391f8
Branch: refs/heads/USERGRID-460
Commit: e70391f8823642f9b5de44e92f06051a74160637
Parents: 67dd8c0 89f558a
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:11:53 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:11:53 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 10 +
.../EntityCollectionManagerFactoryImpl.java | 8 +-
.../impl/EntityCollectionManagerImpl.java | 120 ++++++--
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 +++++++++++++++++--
5 files changed, 382 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
[50/50] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-460
Posted by gr...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-460
# By Todd Nine (24) and Shawn Feldman (16)
# Via Shawn Feldman (7) and others
* 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid: (40 commits)
Added more descriptive error message
Fixes incorrect logging statement
Fixes issue with atomically moving indexes
Fixes bug in writeunique verify failover command
Moved task generation into single factory to clean up code
Updated futures impl for different queues
add comment
add replay strategy
add replay strategy
add replay strategy
Updated queuescope to be name with no need for application id. We are only using them in our subsystems, not applications.
change consistency level
Fixes issue with duplicate pool size definition
Fixes worker issue in cloud formation
Added the ability to specify the number of workers
change metrics prefix
Changes the batch future set.
Changes nodes to 2 per shards with 1 replica.
Fixed naming convention issue
resolve futures via done method
...
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1087ee67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1087ee67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1087ee67
Branch: refs/heads/USERGRID-460
Commit: 1087ee67ac6314c304f42398c357f40e245969a6
Parents: 4964ab6 eb0c689
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 18 13:41:53 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 18 13:41:53 2015 -0700
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 20 +-
.../dist/init_instance/install_elasticsearch.sh | 22 +-
.../main/groovy/configure_elasticsearch.groovy | 80 ++--
.../src/main/groovy/configure_usergrid.groovy | 12 +-
stack/awscluster/ugcluster-cf.json | 466 +++++++++++++------
stack/core/pom.xml | 10 -
.../batch/service/JobSchedulerService.java | 5 -
.../usergrid/corepersistence/CoreModule.java | 8 +-
.../corepersistence/CpEntityManager.java | 15 +-
.../corepersistence/CpEntityManagerFactory.java | 4 -
.../corepersistence/CpRelationManager.java | 14 -
.../events/EntityDeletedHandler.java | 58 ++-
.../events/EntityVersionCreatedHandler.java | 60 ++-
.../events/EntityVersionDeletedHandler.java | 54 ++-
.../results/FilteringLoader.java | 2 +-
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 22 +-
.../cassandra/RelationManagerImpl.java | 58 +--
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
.../corepersistence/StaleIndexCleanupTest.java | 38 +-
.../PerformanceEntityRebuildIndexTest.java | 2 +
.../collection/EntityDeletedFactory.java | 34 --
.../collection/EntityVersionCleanupFactory.java | 35 --
.../collection/EntityVersionCreatedFactory.java | 31 --
.../collection/guice/CollectionModule.java | 22 +-
.../EntityCollectionManagerFactoryImpl.java | 58 ++-
.../impl/EntityCollectionManagerImpl.java | 179 +++++--
.../collection/impl/EntityDeletedTask.java | 21 +-
.../impl/EntityVersionCleanupTask.java | 40 +-
.../impl/EntityVersionTaskFactory.java | 65 +++
.../mvcc/stage/write/WriteUniqueVerify.java | 140 +++---
.../UniqueValueSerializationStrategy.java | 21 +-
.../UniqueValueSerializationStrategyImpl.java | 19 +-
.../collection/util/EntityUtils.java | 28 +-
.../impl/EntityVersionCleanupTaskTest.java | 68 +--
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../persistence/core/future/BetterFuture.java | 43 +-
.../core/metrics/MetricsFactory.java | 9 +
.../core/metrics/MetricsFactoryImpl.java | 121 +++--
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 7 +-
.../astyanax/MultiRowColumnIteratorTest.java | 7 +-
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 ++++++++++--
.../usergrid/persistence/map/MapManager.java | 35 +-
.../persistence/map/impl/MapManagerImpl.java | 8 +
.../persistence/map/impl/MapSerialization.java | 9 +
.../map/impl/MapSerializationImpl.java | 93 ++++
.../persistence/map/MapManagerTest.java | 49 +-
stack/corepersistence/queryindex/pom.xml | 17 +-
.../usergrid/persistence/index/EntityIndex.java | 26 +-
.../persistence/index/IndexBufferConsumer.java | 11 +
.../persistence/index/IndexBufferProducer.java | 1 -
.../usergrid/persistence/index/IndexFig.java | 73 ++-
.../index/IndexOperationMessage.java | 115 ++++-
.../persistence/index/guice/IndexModule.java | 7 +
.../persistence/index/guice/QueueProvider.java | 116 +++++
.../persistence/index/impl/BatchRequest.java | 41 ++
.../persistence/index/impl/BufferQueue.java | 68 +++
.../index/impl/BufferQueueInMemoryImpl.java | 108 +++++
.../index/impl/BufferQueueSQSImpl.java | 307 ++++++++++++
.../persistence/index/impl/DeIndexRequest.java | 115 +++++
.../index/impl/EsEntityIndexBatchImpl.java | 50 +-
.../index/impl/EsEntityIndexImpl.java | 252 +++-------
.../index/impl/EsIndexBufferConsumerImpl.java | 286 ++++++++----
.../index/impl/EsIndexBufferProducerImpl.java | 16 +-
.../persistence/index/impl/EsIndexCache.java | 138 +++---
.../persistence/index/impl/IndexRequest.java | 125 +++++
.../index/guice/TestIndexModule.java | 5 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 +++++++
.../impl/EntityConnectionIndexImplTest.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 62 ++-
.../persistence/index/impl/EsTestUtils.java | 48 --
.../persistence/queue/QueueManager.java | 4 +-
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 --
.../persistence/queue/guice/QueueModule.java | 17 +-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --
.../persistence/queue/impl/QueueScopeImpl.java | 27 +-
.../queue/impl/SQSQueueManagerImpl.java | 214 +++++----
.../persistence/queue/NoAWSCredsRule.java | 98 ++++
.../persistence/queue/QueueManagerTest.java | 29 +-
stack/pom.xml | 43 --
stack/rest/pom.xml | 4 -
.../org/apache/usergrid/rest/RootResource.java | 6 -
stack/services/pom.xml | 13 +-
.../cassandra/ManagementServiceImpl.java | 9 +-
.../notifications/NotificationsService.java | 50 +-
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
.../notifications/NotifiersServiceIT.java | 6 +
stack/test-utils/pom.xml | 5 -
97 files changed, 3634 insertions(+), 1599 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1087ee67/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index ed1f03a,1838eec..5a3c8ce
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -401,24 -393,15 +397,24 @@@ public class EsEntityIndexImpl implemen
}
+ /**
+ * Needs REfactor to make clearer what queries and how the information gets used so we don't need to retraverse
+ * the query every time we generate some information about it.
+ * @param indexScope
+ * @param searchTypes
+ * @param query
+ * @return
+ */
@Override
- public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes,
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName(indexScope);
+ final String context = IndexingUtils.createContextName( indexScope );
final String[] entityTypes = searchTypes.getTypeNames();
- QueryBuilder qb = query.createQueryBuilder(context);
+ final QueryVisitor queryVisitor = query.getQueryVisitor();
+ QueryBuilder qb = query.createQueryBuilder( context );
SearchResponse searchResponse;
[46/50] incubator-usergrid git commit: Fixes incorrect logging
statement
Posted by gr...@apache.org.
Fixes incorrect logging statement
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e6fb121a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e6fb121a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e6fb121a
Branch: refs/heads/USERGRID-460
Commit: e6fb121a8eeabb2ed0aa10b036c2dd0bd1e3f0b4
Parents: c16bb00
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 10:59:31 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 10:59:31 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e6fb121a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cae9cb0..cecd297 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -273,14 +273,14 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//remove the write alias from it's target
for ( String currentIndex : indexNames ) {
aliasesRequestBuilder.removeAlias( currentIndex, alias.getWriteAlias() );
- logger.info("Removing existing write Alias Name [{}] from Index [{}]", alias.getReadAlias(), currentIndex);
+ logger.info("Removing existing write Alias Name [{}] from Index [{}]", alias.getWriteAlias(), currentIndex);
}
//Added For Graphite Metrics
// add read alias
aliasesRequestBuilder.addAlias( indexName, alias.getReadAlias());
- logger.info("Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName);
+ logger.info( "Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName);
//add write alias
[41/50] incubator-usergrid git commit: Merge remote-tracking branch
'origin/USERGRID-466-change-write-consistency' into USERGRID-466
Posted by gr...@apache.org.
Merge remote-tracking branch 'origin/USERGRID-466-change-write-consistency' into USERGRID-466
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/205e0e0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/205e0e0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/205e0e0c
Branch: refs/heads/USERGRID-460
Commit: 205e0e0c03c176623fe9fa2cc18e6a32878838b5
Parents: 2d6ae36 45271d7
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 13:56:33 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 13:56:33 2015 -0600
----------------------------------------------------------------------
.../collection/EntityDeletedFactory.java | 34 -----
.../collection/EntityVersionCleanupFactory.java | 35 -----
.../collection/EntityVersionCreatedFactory.java | 31 -----
.../impl/EntityVersionTaskFactory.java | 65 ++++++++++
.../mvcc/stage/write/WriteUniqueVerify.java | 130 ++++++++++++-------
.../UniqueValueSerializationStrategy.java | 21 ++-
.../UniqueValueSerializationStrategyImpl.java | 19 ++-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 8 +-
.../persistence/core/astyanax/CassandraFig.java | 9 +-
.../core/astyanax/ColumnNameIteratorTest.java | 7 +-
.../MultiKeyColumnNameIteratorTest.java | 7 +-
.../astyanax/MultiRowColumnIteratorTest.java | 7 +-
.../index/impl/EsEntityIndexImpl.java | 31 +++--
15 files changed, 238 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
index 3c673ee,3c673ee..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityDeletedFactory.java
+++ /dev/null
@@@ -1,34 -1,34 +1,0 @@@
--/*
-- *
-- * * Licensed to the Apache Software Foundation (ASF) under one or more
-- * * contributor license agreements. The ASF licenses this file to You
-- * * under the Apache License, Version 2.0 (the "License"); you may not
-- * * use this file except in compliance with the License.
-- * * You may obtain a copy of the License at
-- * *
-- * * http://www.apache.org/licenses/LICENSE-2.0
-- * *
-- * * Unless required by applicable law or agreed to in writing, software
-- * * distributed under the License is distributed on an "AS IS" BASIS,
-- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * * See the License for the specific language governing permissions and
-- * * limitations under the License. For additional information regarding
-- * * copyright in this work, please see the NOTICE file in the top level
-- * * directory of this distribution.
-- *
-- */
--
--package org.apache.usergrid.persistence.collection;
--
--import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--import java.util.UUID;
--
--/**
-- * Creates EntityDeletedTask instances
-- */
--public interface EntityDeletedFactory {
-- public EntityDeletedTask getTask(final CollectionScope collectionScope, final Id entityId, final UUID version );
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
index 2232f00,2232f00..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCleanupFactory.java
+++ /dev/null
@@@ -1,35 -1,35 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one or more
-- * contributor license agreements. The ASF licenses this file to You
-- * under the Apache License, Version 2.0 (the "License"); you may not
-- * use this file except in compliance with the License.
-- * You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing, software
-- * distributed under the License is distributed on an "AS IS" BASIS,
-- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- * See the License for the specific language governing permissions and
-- * limitations under the License. For additional information regarding
-- * copyright in this work, please see the NOTICE file in the top level
-- * directory of this distribution.
-- */
--package org.apache.usergrid.persistence.collection;
--
--
--
--import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--import java.util.UUID;
--
--
--public interface EntityVersionCleanupFactory {
--
-- public EntityVersionCleanupTask getTask(
-- final CollectionScope scope,
-- final Id entityId,
-- final UUID version );
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
index 4248d42,4248d42..0000000
deleted file mode 100644,100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityVersionCreatedFactory.java
+++ /dev/null
@@@ -1,31 -1,31 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * http://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing,
-- * software distributed under the License is distributed on an
-- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-- * KIND, either express or implied. See the License for the
-- * specific language governing permissions and limitations
-- * under the License.
-- */
--package org.apache.usergrid.persistence.collection;
--
--
--import java.util.UUID;
--
--import org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask;
--import org.apache.usergrid.persistence.model.entity.Entity;
--import org.apache.usergrid.persistence.model.entity.Id;
--
--public interface EntityVersionCreatedFactory {
-- public EntityVersionCreatedTask getTask( final CollectionScope scope, final Entity entity);
--
--}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
index 0000000,0000000..b41e668
new file mode 100644
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionTaskFactory.java
@@@ -1,0 -1,0 +1,65 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. The ASF licenses this file to You
++ * under the Apache License, Version 2.0 (the "License"); you may not
++ * use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License. For additional information regarding
++ * copyright in this work, please see the NOTICE file in the top level
++ * directory of this distribution.
++ */
++package org.apache.usergrid.persistence.collection.impl;
++
++
++
++import org.apache.usergrid.persistence.collection.CollectionScope;
++import org.apache.usergrid.persistence.collection.impl.EntityDeletedTask;
++import org.apache.usergrid.persistence.collection.impl.EntityVersionCleanupTask;
++import org.apache.usergrid.persistence.collection.impl.EntityVersionCreatedTask;
++import org.apache.usergrid.persistence.model.entity.Entity;
++import org.apache.usergrid.persistence.model.entity.Id;
++
++import java.util.UUID;
++
++
++public interface EntityVersionTaskFactory {
++
++ /**
++ * Get a task for cleaning up latent entity data. If includeVersion = true, the passed version will be cleaned up as well
++ * Otherwise this is a V-1 operation
++ *
++ * @param scope
++ * @param entityId
++ * @param version
++ * @param includeVersion
++ * @return
++ */
++ public EntityVersionCleanupTask getCleanupTask( final CollectionScope scope, final Id entityId, final UUID version,
++ final boolean includeVersion );
++
++ /**
++ * Get an entityVersionCreatedTask
++ * @param scope
++ * @param entity
++ * @return
++ */
++ public EntityVersionCreatedTask getCreatedTask( final CollectionScope scope, final Entity entity );
++
++ /**
++ * Get an entity deleted task
++ * @param collectionScope
++ * @param entityId
++ * @param version
++ * @return
++ */
++ public EntityDeletedTask getDeleteTask( final CollectionScope collectionScope, final Id entityId,
++ final UUID version );
++
++}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/205e0e0c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8be044f,9343d39..3a11ec7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -169,26 -173,36 +172,36 @@@ public class EsEntityIndexImpl implemen
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
- this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.timer" );
this.removeAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "remove.alias.timer" );
this.addReadAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.read.alias.timer" );
this.addWriteAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" );
+ .getTimer( EsEntityIndexImpl.class, "add.write.alias.timer" );
this.mappingTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" );
+ .getTimer( EsEntityIndexImpl.class, "create.mapping.timer" );
this.refreshTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" );
+ .getTimer( EsEntityIndexImpl.class, "refresh.timer" );
this.searchTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
+ .getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
+ .getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
+ this.allVersionsTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
+ this.deletePreviousTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
+ this.allVersionsTimerFuture = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
+ this.deletePreviousTimerFuture = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
+
+ this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
+
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
mapManager = mapManagerFactory.createMapManager(mapScope);
[24/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/77542c6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/77542c6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/77542c6b
Branch: refs/heads/USERGRID-460
Commit: 77542c6b1bd96f402b33857b453431689ec5b085
Parents: fa0705e ad33ecf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:05:47 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:05:47 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexOperationMessage.java | 4 ++++
.../usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java | 1 +
.../usergrid/persistence/index/impl/BufferQueueSQSImpl.java | 2 +-
3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/77542c6b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
[39/50] incubator-usergrid git commit: Merge branch 'USERGRID-466'
into USERGRID-466-change-write-consistency
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' into USERGRID-466-change-write-consistency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/45271d7a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/45271d7a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/45271d7a
Branch: refs/heads/USERGRID-460
Commit: 45271d7a26ad61063a13307ad567882afe387acc
Parents: 8cea08d 9d00c22
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:24:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:24:16 2015 -0600
----------------------------------------------------------------------
.../main/groovy/configure_elasticsearch.groovy | 1 -
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 31 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
12 files changed, 232 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
[47/50] incubator-usergrid git commit: Merge branch 'USERGRID-490'
into two-dot-o
Posted by gr...@apache.org.
Merge branch 'USERGRID-490' into two-dot-o
* USERGRID-490:
Fixes incorrect logging statement
Fixes issue with atomically moving indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aa044cf7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aa044cf7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aa044cf7
Branch: refs/heads/USERGRID-460
Commit: aa044cf761cd3a3460ed5d91efbc7cf691aab330
Parents: b53cb07 e6fb121
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 18 10:04:43 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 18 10:04:43 2015 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 83 +++++++++-----------
1 file changed, 35 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
[32/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fde59b70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fde59b70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fde59b70
Branch: refs/heads/USERGRID-460
Commit: fde59b709ca242836dccd78e9af6d711a155a130
Parents: 9e6e36d 19c6ad0
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 12 17:29:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 12 17:29:59 2015 -0600
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 8 +-
stack/awscluster/ugcluster-cf.json | 150 ++-----------------
.../usergrid/persistence/index/IndexFig.java | 9 ++
.../index/impl/EsIndexBufferConsumerImpl.java | 55 +++++--
.../index/guice/TestIndexModule.java | 4 +-
5 files changed, 75 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
[11/50] incubator-usergrid git commit: fix pom
Posted by gr...@apache.org.
fix pom
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/89f558ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/89f558ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/89f558ac
Branch: refs/heads/USERGRID-460
Commit: 89f558acdb0c764e1d4f8c7a00cff1b2783d6b2c
Parents: 8ccc3ec
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 11:02:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 11:02:59 2015 -0600
----------------------------------------------------------------------
stack/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/89f558ac/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 73f2cbd..25b2258 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,6 +109,7 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
+ <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
[20/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9630fcf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9630fcf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9630fcf2
Branch: refs/heads/USERGRID-460
Commit: 9630fcf2aaee9e2e56878852d1fad1cf985c6343
Parents: 615e3fc aeb15e9
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 15:23:33 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 15:23:33 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[23/50] incubator-usergrid git commit: Changes nodes to 2 per shards
with 1 replica.
Posted by gr...@apache.org.
Changes nodes to 2 per shards with 1 replica.
Changes ES system properties
Adds remote debugging to tomcat by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fa0705e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fa0705e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fa0705e7
Branch: refs/heads/USERGRID-460
Commit: fa0705e76b001be5e873cb0b1dd1c879d2e7b309
Parents: c594346
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:04:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:04:06 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 20 +++++++++++---------
.../dist/init_instance/install_elasticsearch.sh | 10 ++++------
.../main/groovy/configure_elasticsearch.groovy | 11 +++++++++++
.../src/main/groovy/configure_usergrid.groovy | 2 +-
4 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index cad4e74..ff74851 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -1,14 +1,14 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. The ASF licenses this file to You
# under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,10 +35,10 @@ apt-get -y --force-yes install ${PKGS}
# Install AWS Java SDK and get it into the Groovy classpath
curl http://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip > /tmp/aws-sdk-java.zip
cd /usr/share/
-unzip /tmp/aws-sdk-java.zip
+unzip /tmp/aws-sdk-java.zip
mkdir -p /home/ubuntu/.groovy/lib
cp /usr/share/aws-java-sdk-*/third-party/*/*.jar /home/ubuntu/.groovy/lib
-cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
ln -s /home/ubuntu/.groovy /root/.groovy
# Build environment for Groovy scripts
@@ -55,7 +55,7 @@ groovy tag_instance.groovy -BUILD-IN-PROGRESS
chmod +x /usr/share/usergrid/update.sh
cd /usr/share/usergrid/init_instance
-./install_oraclejdk.sh
+./install_oraclejdk.sh
cd /usr/share/usergrid/init_instance
./install_yourkit.sh
@@ -119,6 +119,8 @@ sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount
#Append our java opts for secret key
echo "JAVA_OPTS=\"\${JAVA_OPTS} -DAWS_SECRET_KEY=${AWS_SECRET_KEY} -DAWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY}\"" >> /etc/default/tomcat7
+echo "JAVA_OPTS=\"\${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000\"" >> /etc/default/tomcat7
+
ulimit -n $NOFILE
@@ -168,7 +170,7 @@ kernel.shmall = 4294967296
######
EOF
-# wait for enough Cassandra nodes then delpoy and configure Usergrid
+# wait for enough Cassandra nodes then delpoy and configure Usergrid
cd /usr/share/usergrid/scripts
groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
@@ -182,8 +184,8 @@ chown -R tomcat7 /usr/share/usergrid/webapps
chown -R tomcat7 /var/lib/tomcat7/webapps
# configure usergrid
-mkdir -p /usr/share/tomcat7/lib
-groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
+mkdir -p /usr/share/tomcat7/lib
+groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index b2018e1..b488320 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -80,8 +80,10 @@ cat >> /etc/default/elasticsearch << EOF
ES_HEAP_SIZE=${ES_HEAP_SIZE}
MAX_OPEN_FILES=65535
MAX_MAP_COUNT=262144
-#MAX_LOCKED_MEMORY=unlimited
+MAX_LOCKED_MEMORY=unlimited
JAVA_HOME=/usr/lib/jvm/jdk1.7.0
+ES_HEAP_NEWSIZE=4g
+ES_JAVA_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:SurvivorRatio=4 -Xloggc:/mnt/raid/elasticsearch/jvm"
EOF
#Set it because Matt says so
@@ -106,15 +108,11 @@ pushd /usr/share/elasticsearch/bin
#Install bigdesk
-./plugin --install lukas-vlcek/bigdesk
-
-./plugin --install mobz/elasticsearch-head
-
./plugin -install royrusso/elasticsearch-HQ
./plugin -install karmi/elasticsearch-paramedic
-./plugin -install xyu/elasticsearch-whatson/0.1.3
+./plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.1
popd
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 19c969d..38b032f 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -26,6 +26,9 @@
//
+String accessKey = (String)System.getenv().get("AWS_ACCESS_KEY")
+String secretKey = (String)System.getenv().get("AWS_SECRET_KEY")
+
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def clusterName = (String)System.getenv().get("ES_CLUSTER_NAME")
@@ -153,6 +156,14 @@ index.indexing.slowlog.threshold.index.info: 5s
index.indexing.slowlog.threshold.index.debug: 2s
index.indexing.slowlog.threshold.index.trace: 500ms
+########
+# AWS PLUGIM
+##########
+
+cloud.aws.access_key: ${accessKey}
+cloud.aws.secret_key: ${secretKey}
+
+
"""
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 9274430..8af5884 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -44,7 +44,7 @@ def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
def numEsNodes = Integer.parseInt(System.getenv().get("ES_NUM_SERVERS"))
//Override number of shards. Set it to 2x the cluster size
-def esShards = numEsNodes;
+def esShards = numEsNodes*2;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
[17/50] incubator-usergrid git commit: Fixes issue with not deleting
from the map on message ack
Posted by gr...@apache.org.
Fixes issue with not deleting from the map on message ack
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8fa5c9f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8fa5c9f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8fa5c9f8
Branch: refs/heads/USERGRID-460
Commit: 8fa5c9f8f4b6580eccba820c5fbbfdf9f088ed87
Parents: 001f3f8
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:52:20 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:52:20 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/BufferQueueSQSImpl.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8fa5c9f8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index c6acb36..25c2ba6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -190,7 +190,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//load them into our response
for ( final QueueMessage message : messages ) {
- final String key = message.getBody().toString();
+ final String key = getMessageKey( message );
//now see if the key was there
final String payload = values.get( key );
@@ -238,6 +238,15 @@ public class BufferQueueSQSImpl implements BufferQueue {
List<QueueMessage> toAck = new ArrayList<>( messages.size() );
for ( IndexOperationMessage ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ final String key = getMessageKey(sqsIndexOperationMessage.getMessage());
+
+ //remove it from the map
+ mapManager.delete( key );
+
toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
}
@@ -257,6 +266,9 @@ public class BufferQueueSQSImpl implements BufferQueue {
return mapper.writeValueAsString( o );
}
+ private String getMessageKey(final QueueMessage message){
+ return message.getBody().toString();
+ }
/**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
[14/50] incubator-usergrid git commit: remove yammer
Posted by gr...@apache.org.
remove yammer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/62635942
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/62635942
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/62635942
Branch: refs/heads/USERGRID-460
Commit: 626359421e9cfedf0fd5baba767051d37b7a24ab
Parents: 89f558a
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 11:35:12 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 11:35:12 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 10 ----
.../batch/service/JobSchedulerService.java | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 4 --
.../corepersistence/CpRelationManager.java | 14 -----
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 23 +-------
.../cassandra/RelationManagerImpl.java | 58 +++-----------------
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
stack/pom.xml | 43 ---------------
stack/rest/pom.xml | 4 --
.../org/apache/usergrid/rest/RootResource.java | 6 --
stack/test-utils/pom.xml | 5 --
15 files changed, 10 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index d30bedb..3a40366 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -279,16 +279,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- </dependency>
-
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
index 5d57ab7..3674383 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/service/JobSchedulerService.java
@@ -46,8 +46,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import com.yammer.metrics.annotation.ExceptionMetered;
-import com.yammer.metrics.annotation.Timed;
/**
@@ -81,8 +79,6 @@ public class JobSchedulerService extends AbstractScheduledService {
public JobSchedulerService() { }
- @Timed( name = "BulkJobScheduledService_runOneIteration", group = "scheduler", durationUnit = TimeUnit.MILLISECONDS,
- rateUnit = TimeUnit.MINUTES )
@Override
protected void runOneIteration() throws Exception {
@@ -143,7 +139,6 @@ public class JobSchedulerService extends AbstractScheduledService {
/**
* Use the provided BulkJobFactory to build and submit BulkJob items as ListenableFuture objects
*/
- @ExceptionMetered( name = "BulkJobScheduledService_submitWork_exceptions", group = "scheduler" )
private void submitWork( final JobDescriptor jobDescriptor ) {
final Job job;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 26eff4c..d808ae5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -108,7 +108,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.netflix.hystrix.exception.HystrixRuntimeException;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -388,7 +387,6 @@ public class CpEntityManager implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_create" )
@TraceParticipant
public <A extends Entity> A create( String entityType, Class<A> entityClass,
Map<String, Object> properties, UUID importId ) throws Exception {
@@ -1261,7 +1259,6 @@ public class CpEntityManager implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValues" )
public Map<String, Object> getDictionaryElementValues( EntityRef entity, String dictionaryName,
String... elementNames ) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 12504ed..fe4d828 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -21,7 +21,6 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import java.util.*;
@@ -391,20 +390,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications() throws Exception {
return getApplications( false );
}
@Override
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getDeletedApplications() throws Exception {
return getApplications( true );
}
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Map<String, UUID> getApplications(boolean deleted) throws Exception {
Map<String, UUID> appMap = new HashMap<String, UUID>();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index d018324..ee81a67 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -108,7 +108,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -490,7 +489,6 @@ public class CpRelationManager implements RelationManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "RelationManager_isOwner" )
@Override
public boolean isCollectionMember( String collName, EntityRef entity ) throws Exception {
@@ -742,7 +740,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createItemInCollection" )
public Entity createItemInCollection(
String collName, String itemType, Map<String, Object> properties) throws Exception {
@@ -1012,7 +1009,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connection_ref" )
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
return createConnection( connection.getConnectionType(), connection.getConnectedEntity() );
@@ -1020,7 +1016,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connectionType" )
public ConnectionRef createConnection( String connectionType, EntityRef connectedEntityRef ) throws Exception {
headEntity = em.validate( headEntity );
@@ -1091,7 +1086,6 @@ public class CpRelationManager implements RelationManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "CpRelationManager_batchUpdateEntityConnection" )
public Mutator<ByteBuffer> batchUpdateEntityConnection(
Mutator<ByteBuffer> batch,
boolean disconnect,
@@ -1200,7 +1194,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_paired_connection_type" )
public ConnectionRef createConnection(
String pairedConnectionType,
EntityRef pairedEntity,
@@ -1212,7 +1205,6 @@ public class CpRelationManager implements RelationManager {
@Override
- @Metered( group = "core", name = "RelationManager_createConnection_connected_entity_ref" )
public ConnectionRef createConnection( ConnectedEntityRef... connections ) throws Exception {
throw new UnsupportedOperationException( "Paired connections not supported" );
@@ -1680,7 +1672,6 @@ public class CpRelationManager implements RelationManager {
* @return The indexUpdate with batch mutations
* @throws Exception the exception
*/
- @Metered( group = "core", name = "RelationManager_batchUpdateCollectionIndex" )
public IndexUpdate batchUpdateCollectionIndex(
IndexUpdate indexUpdate, EntityRef owner, String collectionName )
throws Exception {
@@ -1779,7 +1770,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchStartIndexUpdate")
public IndexUpdate batchStartIndexUpdate(
Mutator<ByteBuffer> batch, Entity entity, String entryName,
Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
@@ -1939,7 +1929,6 @@ public class CpRelationManager implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsDictionaryIndexes")
public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes(
IndexUpdate indexUpdate ) throws Exception {
@@ -2006,7 +1995,6 @@ public class CpRelationManager implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateConnectionIndex")
public IndexUpdate batchUpdateConnectionIndex(
IndexUpdate indexUpdate, ConnectionRefImpl connection ) throws Exception {
@@ -2135,7 +2123,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered( group = "core", name = "RelationManager_batchDeleteConnectionIndexEntries" )
public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries(
IndexUpdate indexUpdate,
IndexUpdate.IndexEntry entry,
@@ -2190,7 +2177,6 @@ public class CpRelationManager implements RelationManager {
}
- @Metered( group = "core", name = "RelationManager_batchAddConnectionIndexEntries" )
public Mutator<ByteBuffer> batchAddConnectionIndexEntries( IndexUpdate indexUpdate, IndexUpdate.IndexEntry entry,
ConnectionRefImpl conn, UUID[] index_keys ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 880fe89..6db05f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
-import com.yammer.metrics.annotation.Metered;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.springframework.context.ApplicationContext;
@@ -97,7 +96,6 @@ public interface EntityManagerFactory {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public abstract Map<String, UUID> getApplications() throws Exception;
public Map<String, UUID> getDeletedApplications() throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index af1eabc..a8c62d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -40,7 +40,6 @@ import org.apache.commons.lang.StringUtils;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -274,7 +273,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
@Override
- @Metered(group = "core", name = "EntityManagerFactory_lookupApplication_byName")
public UUID lookupApplication( String name ) throws Exception {
name = name.toLowerCase();
HColumn<String, ByteBuffer> column =
@@ -295,7 +293,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Application getApplication( String name ) throws Exception {
name = name.toLowerCase();
HColumn<String, ByteBuffer> column =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index be43920..7dcbca7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -90,7 +90,6 @@ import org.apache.usergrid.utils.UUIDUtils;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
@@ -191,6 +190,7 @@ public class EntityManagerImpl implements EntityManager {
public static final String APPLICATION_COLLECTION = "application.collection.";
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
+
@Resource
private EntityManagerFactoryImpl emf;
@Resource
@@ -505,7 +505,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @return True if this entity can safely "own" this property name and value unique combination
*/
- @Metered( group = "core", name = "EntityManager_isPropertyValueUniqueForEntity" )
public boolean isPropertyValueUniqueForEntity( UUID ownerEntityId, String entityType, String propertyName,
Object propertyValue ) throws Exception {
@@ -536,7 +535,6 @@ public class EntityManagerImpl implements EntityManager {
/**
* Return all UUIDs that have this unique value
*
- * @param ownerEntityId The entity id that owns this entity collection
* @param collectionName The entity collection name
* @param propertyName The name of the unique property
* @param propertyValue The value of the unique property
@@ -623,7 +621,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAlias_single" )
public EntityRef getAlias( EntityRef ownerRef, String collectionType, String aliasValue )
throws Exception {
@@ -658,7 +655,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAlias_multi" )
public Map<String, EntityRef> getAlias(
EntityRef ownerRef, String collectionName, List<String> aliases ) throws Exception {
@@ -735,7 +731,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_create" )
@TraceParticipant
public <A extends Entity> A create(
String entityType, Class<A> entityClass, Map<String, Object> properties,
@@ -754,7 +749,6 @@ public class EntityManagerImpl implements EntityManager {
@SuppressWarnings( "unchecked" )
- @Metered( group = "core", name = "EntityManager_batchCreate" )
public <A extends Entity> A batchCreate(
Mutator<ByteBuffer> m, String entityType, Class<A> entityClass,
Map<String, Object> properties, UUID importId, UUID timestampUuid )
@@ -975,7 +969,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_insertEntity" )
public void insertEntity( EntityRef entityRef ) throws Exception {
String type = entityRef.getType();
@@ -1021,7 +1014,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_getEntityType" )
public String getEntityType( UUID entityId ) throws Exception {
HColumn<String, String> column =
@@ -1044,7 +1036,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_loadPartialEntity" )
public DynamicEntity loadPartialEntity( UUID entityId, String... propertyNames ) throws Exception {
List<HColumn<String, ByteBuffer>> results = null;
@@ -1133,7 +1124,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_getEntities" )
public <A extends Entity> List<A> getEntities(
Collection<UUID> entityIds, Class<A> entityClass ) throws Exception {
@@ -1192,7 +1182,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getPropertyNames" )
public Set<String> getPropertyNames( EntityRef entity ) throws Exception {
Set<String> propertyNames = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
@@ -1215,7 +1204,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryNames" )
public Set<String> getDictionaryNames( EntityRef entity ) throws Exception {
Set<String> dictionaryNames = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
@@ -1239,7 +1227,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValue" )
public Object getDictionaryElementValue( EntityRef entity, String dictionaryName, String elementName )
throws Exception {
@@ -1280,7 +1267,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_getDictionaryElementValues" )
public Map<String, Object> getDictionaryElementValues( EntityRef entity, String dictionaryName,
String... elementNames ) throws Exception {
@@ -1342,7 +1328,6 @@ public class EntityManagerImpl implements EntityManager {
* @throws Exception the exception
*/
@Override
- @Metered( group = "core", name = "EntityManager_getDictionaryAsMap" )
public Map<Object, Object> getDictionaryAsMap( EntityRef entity, String dictionaryName ) throws Exception {
entity = validate( entity );
@@ -1405,7 +1390,6 @@ public class EntityManagerImpl implements EntityManager {
*
* @throws Exception the exception
*/
- @Metered( group = "core", name = "EntityManager_updateProperties" )
public void updateProperties(
UUID entityId, String type, Map<String, Object> properties ) throws Exception {
@@ -1423,7 +1407,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Metered( group = "core", name = "EntityManager_deleteEntity" )
public void deleteEntity( UUID entityId, String type ) throws Exception {
logger.info( "deleteEntity {} of application {}", entityId, applicationId );
@@ -1608,7 +1591,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAggregateCounters" )
public Results getAggregateCounters( UUID userId, UUID groupId, UUID queueId, String category,
String counterName, CounterResolution resolution, long start, long finish, boolean pad ) {
@@ -1647,7 +1629,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getAggregateCounters_fromQueryObj" )
public Results getAggregateCounters( Query query ) throws Exception {
CounterResolution resolution = query.getResolution();
if ( resolution == null ) {
@@ -1732,7 +1713,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_getEntityCounters" )
public Map<String, Long> getEntityCounters( UUID entityId ) throws Exception {
Map<String, Long> counters = new HashMap<String, Long>();
@@ -1755,7 +1735,6 @@ public class EntityManagerImpl implements EntityManager {
@Override
- @Metered( group = "core", name = "EntityManager_createApplicationCollection" )
public void createApplicationCollection( String entityType ) throws Exception {
Keyspace ko = cass.getApplicationKeyspace( applicationId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index be84176..c4634c4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -81,7 +81,6 @@ import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
import com.google.common.base.Preconditions;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -210,7 +209,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateCollectionIndex")
public IndexUpdate batchUpdateCollectionIndex( IndexUpdate indexUpdate, EntityRef owner, String collectionName )
throws Exception {
@@ -292,7 +290,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollectionIndexes")
public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
// TODO TN, read all buckets here
@@ -391,7 +388,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchAddToCollections")
public Mutator<ByteBuffer> batchAddToCollections( Mutator<ByteBuffer> batch, String ownerType, List<UUID> ownerIds,
String collectionName, Entity entity, UUID timestampUuid )
throws Exception {
@@ -506,7 +502,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchRemoveFromCollection")
public Mutator<ByteBuffer> batchRemoveFromCollection( Mutator<ByteBuffer> batch, String collectionName,
Entity entity, boolean force, UUID timestampUuid )
throws Exception {
@@ -567,7 +562,7 @@ public class RelationManagerImpl implements RelationManager {
if ( !headEntity.getType().equalsIgnoreCase( TYPE_APPLICATION ) && !Schema
.isAssociatedEntityType( entity.getType() ) ) {
- CollectionRef cref = new SimpleCollectionRef( headEntity, collectionName, entity );
+ CollectionRef cref = new SimpleCollectionRef( headEntity, collectionName, entity );
em.delete( new SimpleEntityRef( cref.getType(), cref.getUuid() ) );
}
@@ -575,7 +570,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchDeleteConnectionIndexEntries")
public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries( IndexUpdate indexUpdate, IndexEntry entry,
ConnectionRefImpl connection, UUID[] index_keys )
throws Exception {
@@ -626,7 +620,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchAddConnectionIndexEntries")
public Mutator<ByteBuffer> batchAddConnectionIndexEntries( IndexUpdate indexUpdate, IndexEntry entry,
ConnectionRefImpl connection, UUID[] index_keys ) {
@@ -687,7 +680,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateConnectionIndex")
public IndexUpdate batchUpdateConnectionIndex( IndexUpdate indexUpdate, ConnectionRefImpl connection )
throws Exception {
@@ -733,7 +725,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* addInsertToMutator(batch, EntityCF.SETS, key(connection_id,
* Schema.INDEXES_SET), indexEntry.getKey(), null, false, timestamp); }
- *
+ *
* addInsertToMutator(batch, EntityCF.SETS, key(connection_id,
* Schema.INDEXES_SET), entryName, null, false, timestamp);
*/
@@ -775,7 +767,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsPropertyIndexes")
public IndexUpdate batchUpdateBackwardConnectionsPropertyIndexes( IndexUpdate indexUpdate ) throws Exception {
logger.debug( "batchUpdateBackwordConnectionsPropertyIndexes" );
@@ -851,7 +842,6 @@ public class RelationManagerImpl implements RelationManager {
*
* @throws Exception the exception
*/
- @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsDictionaryIndexes")
public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes( IndexUpdate indexUpdate ) throws Exception {
logger.debug( "batchUpdateBackwardConnectionsListIndexes" );
@@ -869,13 +859,12 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
- @Metered(group = "core", name = "RelationManager_batchUpdateEntityConnection")
- public Mutator<ByteBuffer> batchUpdateEntityConnection( Mutator<ByteBuffer> batch,
+ public Mutator<ByteBuffer> batchUpdateEntityConnection( Mutator<ByteBuffer> batch,
boolean disconnect, ConnectionRefImpl connection, UUID timestampUuid ) throws Exception {
long timestamp = getTimestampInMicros( timestampUuid );
- Entity connectedEntity = em.get( new SimpleEntityRef(
+ Entity connectedEntity = em.get( new SimpleEntityRef(
connection.getConnectedEntityType(), connection.getConnectedEntityId()) );
if ( connectedEntity == null ) {
@@ -1072,7 +1061,7 @@ public class RelationManagerImpl implements RelationManager {
ConnectionRefImpl loopback = connection.getConnectionToConnectionEntity();
if ( !disconnect ) {
- em.insertEntity( new SimpleEntityRef(
+ em.insertEntity( new SimpleEntityRef(
CONNECTION_ENTITY_CONNECTION_TYPE, loopback.getConnectedEntityId() ) );
}
@@ -1082,7 +1071,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchDisconnect")
public void batchDisconnect( Mutator<ByteBuffer> batch, UUID timestampUuid ) throws Exception {
@@ -1120,7 +1108,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchStartIndexUpdate")
public IndexUpdate batchStartIndexUpdate( Mutator<ByteBuffer> batch, Entity entity, String entryName,
Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
boolean isMultiValue, boolean removeListEntry, boolean fulltextIndexed,
@@ -1250,7 +1237,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchUpdatePropertyIndexes")
public void batchUpdatePropertyIndexes( Mutator<ByteBuffer> batch, String propertyName, Object propertyValue,
boolean entitySchemaHasProperty, boolean noRead, UUID timestampUuid )
throws Exception {
@@ -1416,7 +1402,6 @@ public class RelationManagerImpl implements RelationManager {
@SuppressWarnings("unchecked")
@Override
- @Metered(group = "core", name = "RelationManager_isOwner")
public boolean isCollectionMember( String collectionName, EntityRef entity ) throws Exception {
Keyspace ko = cass.getApplicationKeyspace( applicationId );
@@ -1475,7 +1460,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getOwners")
public Map<String, Map<UUID, Set<String>>> getOwners() throws Exception {
Map<EntityRef, Set<String>> containerEntities = getContainingCollections();
Map<String, Map<UUID, Set<String>>> owners = new LinkedHashMap<String, Map<UUID, Set<String>>>();
@@ -1492,7 +1476,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollections")
public Set<String> getCollections() throws Exception {
Map<String, CollectionInfo> collections = getDefaultSchema().getCollections( headEntity.getType() );
@@ -1505,7 +1488,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollection_start_result")
public Results getCollection( String collectionName, UUID startResult, int count, Level resultsLevel,
boolean reversed ) throws Exception {
// changed intentionally to delegate to search so that behavior is
@@ -1524,7 +1506,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getCollecitonForQuery")
public Results getCollection( String collectionName, Query query, Level resultsLevel ) throws Exception {
// changed intentionally to delegate to search so that behavior is
@@ -1535,7 +1516,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_addToCollection")
public Entity addToCollection( String collectionName, EntityRef itemRef ) throws Exception {
Entity itemEntity = em.get( itemRef );
@@ -1566,7 +1546,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_addToCollections")
public Entity addToCollections( List<EntityRef> owners, String collectionName ) throws Exception {
Entity itemEntity = getHeadEntity();
@@ -1600,7 +1579,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createItemInCollection")
public Entity createItemInCollection( String collectionName, String itemType, Map<String, Object> properties )
throws Exception {
@@ -1653,7 +1631,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_removeFromCollection")
public void removeFromCollection( String collectionName, EntityRef itemRef ) throws Exception {
if ( headEntity.getUuid().equals( applicationId ) ) {
@@ -1703,7 +1680,6 @@ public class RelationManagerImpl implements RelationManager {
}
- @Metered(group = "core", name = "RelationManager_batchRemoveFromContainers")
public void batchRemoveFromContainers( Mutator<ByteBuffer> m, UUID timestampUuid ) throws Exception {
Entity entity = getHeadEntity();
// find all the containing collections
@@ -1720,7 +1696,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_copyRelationships")
public void copyRelationships( String srcRelationName, EntityRef dstEntityRef, String dstRelationName )
throws Exception {
@@ -1757,7 +1732,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_searchCollection")
public Results searchCollection( String collectionName, Query query ) throws Exception {
if ( query == null ) {
@@ -1782,7 +1756,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connection_ref")
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
ConnectionRefImpl connectionImpl = new ConnectionRefImpl( connection );
@@ -1793,7 +1766,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connectionType")
public ConnectionRef createConnection( String connectionType, EntityRef connectedEntityRef ) throws Exception {
headEntity = em.validate( headEntity );
@@ -1808,7 +1780,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_paired_connection_type")
public ConnectionRef createConnection( String pairedConnectionType, EntityRef pairedEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
@@ -1823,7 +1794,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_createConnection_connected_entity_ref")
public ConnectionRef createConnection( ConnectedEntityRef... connections ) throws Exception {
ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connections );
@@ -1835,7 +1805,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_type_entity")
public ConnectionRef connectionRef( String connectionType, EntityRef connectedEntityRef ) throws Exception {
ConnectionRef connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
@@ -1845,7 +1814,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_entity_to_entity")
public ConnectionRef connectionRef( String pairedConnectionType, EntityRef pairedEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
@@ -1858,7 +1826,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_connectionRef_connections")
public ConnectionRef connectionRef( ConnectedEntityRef... connections ) {
ConnectionRef connection = new ConnectionRefImpl( headEntity, connections );
@@ -1868,14 +1835,12 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_deleteConnection")
public void deleteConnection( ConnectionRef connectionRef ) throws Exception {
updateEntityConnection( true, new ConnectionRefImpl( connectionRef ) );
}
@Override
- @Metered(group = "core", name = "RelationManager_getConnectionTypes_entity_id")
public Set<String> getConnectionTypes( UUID connectedEntityId ) throws Exception {
// Add connection type to connections set
// addInsertToMutator(batch, ENTITY_DICTIONARIES,
@@ -1923,7 +1888,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectionTypes")
public Set<String> getConnectionTypes( boolean filterConnection ) throws Exception {
Set<String> connections = cast( em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
if ( connections == null ) {
@@ -1937,7 +1901,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectedEntities")
public Results getConnectedEntities( String connectionType, String connectedEntityType, Level resultsLevel )
throws Exception {
@@ -1976,7 +1939,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectingEntities")
public Results getConnectingEntities( String connectionType, String connectedEntityType,
Level resultsLevel ) throws Exception {
@@ -1985,7 +1947,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_getConnectingEntities")
public Results getConnectingEntities(String connectionType,
String entityType, Level level, int count) throws Exception {
return getConnectingEntities(headEntity, connectionType, entityType, level, count );
@@ -2032,7 +1993,6 @@ public class RelationManagerImpl implements RelationManager {
@Override
- @Metered(group = "core", name = "RelationManager_searchConnectedEntities")
public Results searchConnectedEntities( Query query ) throws Exception {
Preconditions.checkNotNull(query, "Query must not be null");
@@ -2143,7 +2103,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
* persistence.query.ir.WithinNode)
*/
@@ -2165,7 +2125,7 @@ public class RelationManagerImpl implements RelationManager {
@Override
public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
- EntityRef ref = em.getAlias(
+ EntityRef ref = em.getAlias(
headEntity, collection.getType(), nameIdentifierNode.getName() );
if ( ref == null ) {
@@ -2236,7 +2196,7 @@ public class RelationManagerImpl implements RelationManager {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.usergrid.persistence.query.ir.NodeVisitor#visit(org.apache.usergrid.
* persistence.query.ir.WithinNode)
*/
@@ -2324,7 +2284,7 @@ public class RelationManagerImpl implements RelationManager {
@Override
public void visit( NameIdentifierNode nameIdentifierNode ) throws Exception {
//TODO T.N. USERGRID-1919 actually validate this is connected
- EntityRef ref = em.getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ EntityRef ref = em.getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
connection.getConnectedEntityType(), nameIdentifierNode.getName() );
if ( ref == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index b412df8..ed705b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -27,7 +27,6 @@ import java.util.UUID;
import org.springframework.util.Assert;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -249,7 +248,6 @@ public class ConnectedIndexScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- @Metered( group = "core", name = "IndexBucketScanner_load" )
public Set<HColumn<ByteBuffer, ByteBuffer>> next() {
Set<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index b2ca591..56c651b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -31,7 +31,6 @@ import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import com.yammer.metrics.annotation.Metered;
import me.prettyprint.hector.api.beans.HColumn;
@@ -209,7 +208,6 @@ public class IndexBucketScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- @Metered(group = "core", name = "IndexBucketScanner_load")
public NavigableSet<HColumn<ByteBuffer, ByteBuffer>> next() {
NavigableSet<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 25b2258..ee3f9cd 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,7 +109,6 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
@@ -826,12 +825,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- <version>${metrics-version}</version>
- </dependency>
-
- <dependency>
<groupId>com.github.stephenc</groupId>
<artifactId>jamm</artifactId>
<version>0.2.5</version>
@@ -1072,44 +1065,8 @@
<version>${snakeyaml-version}</version>
</dependency>
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics-version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-spring</artifactId>
- <version>${metrics-version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
- </exclusion>
-
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-aop</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-beans</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>jline</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 5a856de..deb11f8 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -259,10 +259,6 @@
<artifactId>jackson-jaxrs-json-provider</artifactId>
</dependency>
- <dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-spring</artifactId>
- </dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
index 27eff28..e4086f9 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
@@ -55,8 +55,6 @@ import org.apache.shiro.authz.UnauthorizedException;
import com.google.common.collect.BiMap;
import com.sun.jersey.api.json.JSONWithPadding;
import com.yammer.metrics.Metrics;
-import com.yammer.metrics.annotation.ExceptionMetered;
-import com.yammer.metrics.annotation.Timed;
import com.yammer.metrics.core.Counter;
import com.yammer.metrics.core.Gauge;
import com.yammer.metrics.core.Histogram;
@@ -297,8 +295,6 @@ public class RootResource extends AbstractContextResource implements MetricProce
public static final String ENTITY_ID_PATH = "{entityId: " + Identifier.UUID_REX + "}";
public static final String EMAIL_PATH = "{email: " + Identifier.EMAIL_REX + "}";
- @Timed(name = "getApplicationByUuids_timer", group = "rest_timers")
- @ExceptionMetered(group = "rest_exceptions", name = "getApplicationByUuids_exceptions")
@Path(ORGANIZATION_ID_PATH+"/"+APPLICATION_ID_PATH)
public ApplicationResource getApplicationByUuids( @PathParam("organizationId") String organizationIdStr,
@PathParam("applicationId") String applicationIdStr )
@@ -324,8 +320,6 @@ public class RootResource extends AbstractContextResource implements MetricProce
}
- @Timed(name = "getOrganizationByName_timer", group = "rest_timers")
- @ExceptionMetered(group = "rest_exceptions", name = "getOrganizationByName_exceptions")
@Path("{organizationName}")
public OrganizationResource getOrganizationByName( @PathParam("organizationName") String organizationName )
throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/62635942/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index fecf069..cf82373 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -149,11 +149,6 @@
</dependency>
<dependency>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
[42/50] incubator-usergrid git commit: Moved task generation into
single factory to clean up code
Posted by gr...@apache.org.
Moved task generation into single factory to clean up code
Fixes repair by version
Fixes incorrect task invocation during writes
Fixes swallowed exception bug
Adds on error processing to QueueConsumer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f20dfd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f20dfd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f20dfd3c
Branch: refs/heads/USERGRID-460
Commit: f20dfd3cbee1f3abb09ae04036b9e747d2485397
Parents: 205e0e0
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 13:57:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 13:57:24 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +
.../events/EntityDeletedHandler.java | 58 ++++++++++-------
.../events/EntityVersionCreatedHandler.java | 58 +++++++++++------
.../events/EntityVersionDeletedHandler.java | 54 +++++++++-------
.../corepersistence/StaleIndexCleanupTest.java | 31 +++++++--
.../PerformanceEntityRebuildIndexTest.java | 2 +
.../collection/guice/CollectionModule.java | 22 ++-----
.../EntityCollectionManagerFactoryImpl.java | 54 +++++++---------
.../impl/EntityCollectionManagerImpl.java | 40 +++++-------
.../collection/impl/EntityDeletedTask.java | 21 +++---
.../impl/EntityVersionCleanupTask.java | 41 ++++++------
.../impl/EntityVersionCleanupTaskTest.java | 68 +++++++++++---------
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 13 ----
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
15 files changed, 252 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index d808ae5..e2b7bd1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -620,6 +620,8 @@ public class CpEntityManager implements EntityManager {
WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause();
handleWriteUniqueVerifyException( entity, wuve );
}
+
+ throw hre;
}
// update in all containing collections and connection indexes
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 33cc988..94ef5e4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -17,52 +17,66 @@
*/
package org.apache.usergrid.corepersistence.events;
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.index.EntityIndex;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
/**
* Delete all Query Index indexes associated with an Entity that has just been deleted.
*/
+@Singleton
public class EntityDeletedHandler implements EntityDeleted {
- private static final Logger logger = LoggerFactory.getLogger(EntityDeletedHandler.class );
+ private static final Logger logger = LoggerFactory.getLogger( EntityDeletedHandler.class );
+
+
+ private final EntityManagerFactory emf;
+
@Inject
- EntityManagerFactory emf;
+ public EntityDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
@Override
- public void deleted(CollectionScope scope, Id entityId, UUID version) {
+ public void deleted( CollectionScope scope, Id entityId, UUID version ) {
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
- if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+ if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" ) ) {
return;
}
- logger.debug("Handling deleted event for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
+ logger.debug( "Handling deleted event for entity {}:{} v {} " + "scope\n name: {}\n owner: {}\n app: {}",
new Object[] {
- entityId.getType(),
- entityId.getUuid(),
- version,
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
+ entityId.getType(), entityId.getUuid(), version, scope.getName(), scope.getOwner(),
+ scope.getApplication()
+ } );
+
+ CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf;
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
+
+ final IndexScope indexScope =
+ new IndexScopeImpl( new SimpleId( scope.getOwner().getUuid(), scope.getOwner().getType() ),
+ scope.getName() );
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
-// ei.deleteAllVersionsOfEntity( entityId );
+ ei.createBatch().deindex( indexScope, entityId, version ).execute();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 590df61..6270501 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.corepersistence.events;
import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,6 +27,7 @@ import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -35,34 +38,51 @@ import org.apache.usergrid.persistence.model.entity.Entity;
* updated by the Collections module and we react by calling the Query Index module and removing
* any indexes that exist for previous versions of the the Entity.
*/
+@Singleton
public class EntityVersionCreatedHandler implements EntityVersionCreated {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
+
+ private final EntityManagerFactory emf;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
@Inject
- EntityManagerFactory emf;
+ public EntityVersionCreatedHandler( final EntityManagerFactory emf,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.emf = emf;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
@Override
public void versionCreated( final CollectionScope scope, final Entity entity ) {
+ //not op, we're not migrating properly to this. Make this an event
- // This check is for testing purposes and for a test that to be able to dynamically turn
- // off and on delete previous versions so that it can test clean-up on read.
- if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
- return;
- }
-
- logger.debug("Handling versionCreated for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] {
- entity.getId().getType(),
- entity.getId().getUuid(),
- entity.getVersion(),
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
-
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+// // This check is for testing purposes and for a test that to be able to dynamically turn
+// // off and on delete previous versions so that it can test clean-up on read.
+// if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+// return;
+// }
+//
+// logger.debug("Handling versionCreated for entity {}:{} v {} "
+// + "scope\n name: {}\n owner: {}\n app: {}",
+// new Object[] {
+// entity.getId().getType(),
+// entity.getId().getUuid(),
+// entity.getVersion(),
+// scope.getName(),
+// scope.getOwner(),
+// scope.getApplication()});
+//
+// CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
+// final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+//
+//
+//
+//
+//
+//
// ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 127ae46..01848cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.corepersistence.events;
import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
import java.util.List;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import static org.apache.usergrid.corepersistence.CoreModule.EVENTS_DISABLED;
@@ -34,6 +36,10 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Action2;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -43,14 +49,17 @@ import rx.schedulers.Schedulers;
* TODO: do we need this? Don't our version-created and entity-deleted handlers take care of this?
* If we do need it then it should be wired in via GuiceModule in the corepersistence package.
*/
+@Singleton
public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionDeletedHandler.class );
- @Inject
- private SerializationFig serializationFig;
+
+
+
+ private final EntityManagerFactory emf;
@Inject
- private EntityManagerFactory emf;
+ public EntityVersionDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
@Override
@@ -63,40 +72,35 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
return;
}
- logger.debug("Handling versionDeleted count={} event for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] {
- entityVersions.size(),
- entityId.getType(),
- entityId.getUuid(),
- scope.getName(),
- scope.getOwner(),
- scope.getApplication()});
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Handling versionDeleted count={} event for entity {}:{} v {} " + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] {
+ entityVersions.size(), entityId.getType(), entityId.getUuid(), scope.getName(), scope.getOwner(),
+ scope.getApplication()
+ } );
+ }
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- final EntityIndexBatch eibatch = ei.createBatch();
-
final IndexScope indexScope = new IndexScopeImpl(
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
scope.getName()
);
- rx.Observable.from(entityVersions)
- .subscribeOn(Schedulers.io())
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ Observable.from( entityVersions )
+ .collect( ei.createBatch(), new Action2<EntityIndexBatch, MvccEntity>() {
@Override
- public List<MvccEntity> call(List<MvccEntity> entityList) {
- for (MvccEntity entity : entityList) {
- eibatch.deindex(indexScope, entityId, entity.getVersion());
- }
- eibatch.execute();
- return entityList;
+ public void call( final EntityIndexBatch entityIndexBatch, final MvccEntity mvccEntity ) {
+ entityIndexBatch.deindex( indexScope, mvccEntity.getId(), mvccEntity.getVersion() );
}
- }).toBlocking().last();
+ } ).doOnNext( new Action1<EntityIndexBatch>() {
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch ) {
+ entityIndexBatch.execute();
+ }
+ } ).toBlocking().last();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 8d31b69..ac04dcc 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -246,7 +246,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
*/
- @Test(timeout=30000)
+// @Test(timeout=30000)
+ @Test
public void testCleanupOnDelete() throws Exception {
logger.info("Started testStaleIndexCleanup()");
@@ -316,11 +317,19 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// wait for indexes to be cleared for the deleted entities
count = 0;
+
+
+ //we can't use our candidate result sets here. The repair won't happen since we now have orphaned documents in our index
+ //us the EM so the repair process happens
+
+ Results results = null;
do {
- Thread.sleep(100);
+ //trigger the repair
+ results = queryCollectionEm("things", "select *");
crs = queryCollectionCp("things", "thing", "select *");
- em.refreshIndex();
- } while ( crs.size() > 0 && count++ < 15 );
+ Thread.sleep(100);
+
+ } while ((results.hasCursor() || crs.size() > 0) && count++ < 2000 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
@@ -436,4 +445,18 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
return ei.search( is, SearchTypes.fromTypes( type ), rcq );
}
+
+ /**
+ * Go around EntityManager and execute query directly against Core Persistence.
+ * Results may include stale index entries.
+ */
+ private Results queryCollectionEm( final String collName, final String query ) throws Exception {
+
+ EntityManager em = app.getEntityManager();
+
+
+ final Results results = em.searchCollection( em.getApplicationRef(), collName, Query.fromQL( query ).withLimit( 10000 ) );
+
+ return results;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index def9ed5..f1f165d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -362,6 +362,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
registry.remove( meterName );
logger.info("Rebuilt index");
+ setup.getEmf().refreshIndex();
+
} catch (Exception ex) {
logger.error("Error rebuilding index", ex);
fail();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index 7cd383a..a73d7a7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -18,30 +18,24 @@
package org.apache.usergrid.persistence.collection.guice;
-
import org.safehaus.guicyfig.GuicyFigModule;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+import org.apache.usergrid.persistence.collection.impl.EntityVersionTaskFactory;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerFactoryImpl;
-import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerImpl;
-import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerSyncImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.SerializationModule;
+import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.service.impl.ServiceModule;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -52,8 +46,6 @@ import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.model.entity.Entity;
/**
@@ -72,9 +64,7 @@ public class CollectionModule extends AbstractModule {
install( new SerializationModule() );
install( new ServiceModule() );
- install ( new FactoryModuleBuilder().build( EntityVersionCleanupFactory.class ));
- install ( new FactoryModuleBuilder().build( EntityDeletedFactory.class));
- install ( new FactoryModuleBuilder().build( EntityVersionCreatedFactory.class ));
+ install ( new FactoryModuleBuilder().build( EntityVersionTaskFactory.class ));
// users of this module can add their own implemementations
// for more information: https://github.com/google/guice/wiki/Multibindings
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index bb1b56a..23e375d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -19,23 +19,6 @@
package org.apache.usergrid.persistence.collection.impl;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.netflix.astyanax.Keyspace;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.guice.Write;
-import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
-import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import java.util.concurrent.ExecutionException;
@@ -43,17 +26,32 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerSync;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
import org.apache.usergrid.persistence.collection.cache.CachedEntityCollectionManager;
import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+import org.apache.usergrid.persistence.collection.guice.Write;
+import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
+import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
@@ -76,9 +74,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final Keyspace keyspace;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
@@ -92,8 +88,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final EntityCollectionManager target = new EntityCollectionManagerImpl( writeStart, writeUpdate, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace, entityVersionCleanupFactory,
- entityVersionCreatedFactory, entityDeletedFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace, entityVersionTaskFactory,
+ taskExecutor, scope, metricsFactory );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -115,9 +111,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
final EntityCacheFig entityCacheFig,
MetricsFactory metricsFactory) {
@@ -134,9 +128,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
this.metricsFactory = metricsFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 50b581a..92b07f0 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,9 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,6 +31,7 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@ -47,11 +45,13 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -59,6 +59,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -68,12 +70,6 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.core.task.Task;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Notification;
import rx.Observable;
@@ -112,9 +108,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
- private final EntityVersionCreatedFactory entityVersionCreatedFactory;
- private final EntityDeletedFactory entityDeletedFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
private final Keyspace keyspace;
@@ -144,9 +138,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
+ final EntityVersionTaskFactory entityVersionTaskFactory,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
@Assisted final CollectionScope collectionScope,
final MetricsFactory metricsFactory
@@ -170,9 +162,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.keyspace = keyspace;
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.entityDeletedFactory = entityDeletedFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.taskExecutor = taskExecutor;
this.collectionScope = collectionScope;
@@ -216,8 +206,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public void call(final Entity entity) {
//TODO fire the created task first then the entityVersioncleanup
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
- taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
+ taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
+ taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entityId,
+ entity.getVersion(), false ));
//post-processing to come later. leave it empty for now.
}
}).doOnError(rollback)
@@ -252,8 +243,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityDeletedFactory
- .getTask(collectionScope, entity.getId(), entity.getVersion());
+ Task<Void> task = entityVersionTaskFactory
+ .getDeleteTask( collectionScope, entity.getId(), entity.getVersion() );
taskExecutor.submit(task);
return entity.getId();
}
@@ -400,8 +391,9 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
logger.debug("sending entity to the queue");
//we an update, signal the fix
- taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope, entity));
+ taskExecutor.submit( entityVersionTaskFactory.getCreatedTask( collectionScope, entity ));
+ taskExecutor.submit( entityVersionTaskFactory.getCleanupTask( collectionScope, entity.getId(), entity.getVersion(), false) );
//TODO T.N Change this to fire a task
// Observable.from( new CollectionIoEvent<Id>(collectionScope,
// entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index 9ff4f56..83f165d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@ -22,7 +22,6 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
@@ -46,7 +45,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
public class EntityDeletedTask implements Task<Void> {
private static final Logger LOG = LoggerFactory.getLogger(EntityDeletedTask.class);
- private final EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final EntityVersionTaskFactory entityVersionTaskFactory;
private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
private final Set<EntityDeleted> listeners;
@@ -56,16 +55,16 @@ public class EntityDeletedTask implements Task<Void> {
@Inject
- public EntityDeletedTask(
- EntityVersionCleanupFactory entityVersionCleanupFactory,
+ public EntityDeletedTask(
+ EntityVersionTaskFactory entityVersionTaskFactory,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
@ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
final Set<EntityDeleted> listeners, // MUST be a set or Guice will not inject
- @Assisted final CollectionScope collectionScope,
- @Assisted final Id entityId,
+ @Assisted final CollectionScope collectionScope,
+ @Assisted final Id entityId,
@Assisted final UUID version) {
- this.entityVersionCleanupFactory = entityVersionCleanupFactory;
+ this.entityVersionTaskFactory = entityVersionTaskFactory;
this.logEntrySerializationStrategy = logEntrySerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
this.listeners = listeners;
@@ -81,7 +80,7 @@ public class EntityDeletedTask implements Task<Void> {
new Object[] { collectionScope, entityId, version }, throwable );
}
-
+
@Override
public Void rejected() {
try {
@@ -94,11 +93,11 @@ public class EntityDeletedTask implements Task<Void> {
return null;
}
-
+
@Override
- public Void call() throws Exception {
+ public Void call() throws Exception {
- entityVersionCleanupFactory.getTask( collectionScope, entityId, version ).call();
+ entityVersionTaskFactory.getCleanupTask( collectionScope, entityId, version, true ).call();
fireEvents();
final MutationBatch entityDelete = entitySerializationStrategy.delete(collectionScope, entityId, version);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index efecdeb..03a5ff6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -55,7 +55,7 @@ import rx.schedulers.Schedulers;
/**
- * Cleans up previous versions from the specified version. Note that this means the version
+ * Cleans up previous versions from the specified version. Note that this means the version
* passed in the io event is retained, the range is exclusive.
*/
public class EntityVersionCleanupTask implements Task<Void> {
@@ -74,10 +74,11 @@ public class EntityVersionCleanupTask implements Task<Void> {
private final CollectionScope scope;
private final Id entityId;
private final UUID version;
+ private final int numToSkip;
@Inject
- public EntityVersionCleanupTask(
+ public EntityVersionCleanupTask(
final SerializationFig serializationFig,
final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
@ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
@@ -86,7 +87,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
@Assisted final CollectionScope scope,
@Assisted final Id entityId,
- @Assisted final UUID version ) {
+ @Assisted final UUID version,
+ @Assisted final boolean includeVersion) {
this.serializationFig = serializationFig;
this.logEntrySerializationStrategy = logEntrySerializationStrategy;
@@ -97,6 +99,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
this.scope = scope;
this.entityId = entityId;
this.version = version;
+
+ numToSkip = includeVersion? 0: 1;
}
@@ -136,7 +140,7 @@ public class EntityVersionCleanupTask implements Task<Void> {
}
})
//buffer them for efficiency
- .skip(1)
+ .skip(numToSkip)
.buffer(serializationFig.getBufferSize()).doOnNext(
new Action1<List<MvccEntity>>() {
@Override
@@ -146,27 +150,28 @@ public class EntityVersionCleanupTask implements Task<Void> {
final MutationBatch logBatch = keyspace.prepareMutationBatch();
for (MvccEntity mvccEntity : mvccEntities) {
- if (!mvccEntity.getEntity().isPresent()) {
- continue;
- }
-
final UUID entityVersion = mvccEntity.getVersion();
- final Entity entity = mvccEntity.getEntity().get();
- //remove all unique fields from the index
- for (final Field field : entity.getFields()) {
- if (!field.isUnique()) {
- continue;
+
+ //if the entity is present process the fields
+ if(mvccEntity.getEntity().isPresent()) {
+ final Entity entity = mvccEntity.getEntity().get();
+
+ //remove all unique fields from the index
+ for ( final Field field : entity.getFields() ) {
+ if ( !field.isUnique() ) {
+ continue;
+ }
+ final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
+ final MutationBatch deleteMutation =
+ uniqueValueSerializationStrategy.delete( scope, unique );
+ batch.mergeShallow( deleteMutation );
}
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion);
- final MutationBatch deleteMutation =
- uniqueValueSerializationStrategy.delete(scope,unique);
- batch.mergeShallow(deleteMutation);
}
final MutationBatch entityDelete = entitySerializationStrategy
.delete(scope, entityId, mvccEntity.getVersion());
- entityBatch.mergeShallow(entityDelete);
+ entityBatch.mergeShallow( entityDelete );
final MutationBatch logDelete = logEntrySerializationStrategy
.delete(scope, entityId, version);
logBatch.mergeShallow(logDelete);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index d0a87c3..f2fb95b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -79,7 +79,7 @@ public class EntityVersionCleanupTaskTest {
@Test(timeout=10000)
- public void noListenerOneVersion()
+ public void noListenerOneVersion()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -109,7 +109,7 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
@@ -133,7 +133,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch newBatch = mock( MutationBatch.class );
@@ -148,10 +149,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory(
@@ -175,7 +176,7 @@ public class EntityVersionCleanupTaskTest {
* Tests the cleanup task on the first version created
*/
@Test(timeout=10000)
- public void noListenerNoVersions()
+ public void noListenerNoVersions()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -208,14 +209,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, 1 );
@@ -233,7 +234,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -249,10 +251,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -267,7 +269,7 @@ public class EntityVersionCleanupTaskTest {
// These last two verify statements do not make sense. We cannot assert that the entity
- // and log batches are never called. Even if there are no listeners the entity delete
+ // and log batches are never called. Even if there are no listeners the entity delete
// cleanup task will still run to do the normal cleanup.
//
// verify( entityBatch, never() ).execute();
@@ -276,7 +278,7 @@ public class EntityVersionCleanupTaskTest {
@Test(timeout=10000)
- public void singleListenerSingleVersion()
+ public void singleListenerSingleVersion()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -317,14 +319,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
@@ -343,7 +345,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -361,10 +364,10 @@ public class EntityVersionCleanupTaskTest {
final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -421,7 +424,7 @@ public class EntityVersionCleanupTaskTest {
final int sizeToReturn = 10;
- final CountDownLatch latch = new CountDownLatch(
+ final CountDownLatch latch = new CountDownLatch(
sizeToReturn/serializationFig.getBufferSize() * 3 );
final EntityVersionDeletedTest listener1 = new EntityVersionDeletedTest( latch );
@@ -436,13 +439,13 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
// mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
@@ -456,7 +459,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -474,10 +478,10 @@ public class EntityVersionCleanupTaskTest {
Entity entity = new Entity( entityId );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.of(entity)) );
- mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
MvccEntity.Status.DELETED, Optional.of(entity)) );
when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
@@ -543,7 +547,7 @@ public class EntityVersionCleanupTaskTest {
final int listenerCount = 5;
- final CountDownLatch latch = new CountDownLatch(
+ final CountDownLatch latch = new CountDownLatch(
sizeToReturn/serializationFig.getBufferSize() * listenerCount );
final Semaphore waitSemaphore = new Semaphore( 0 );
@@ -565,14 +569,14 @@ public class EntityVersionCleanupTaskTest {
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
@@ -591,7 +595,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -714,7 +719,8 @@ public class EntityVersionCleanupTaskTest {
listeners,
appScope,
entityId,
- version
+ version,
+ false
);
final MutationBatch batch = mock( MutationBatch.class );
@@ -768,7 +774,7 @@ public class EntityVersionCleanupTaskTest {
@Override
- public void versionDeleted( final CollectionScope scope, final Id entityId,
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
final List<MvccEntity> entityVersion ) {
invocationLatch.countDown();
}
@@ -786,7 +792,7 @@ public class EntityVersionCleanupTaskTest {
@Override
- public void versionDeleted( final CollectionScope scope, final Id entityId,
+ public void versionDeleted( final CollectionScope scope, final Id entityId,
final List<MvccEntity> entityVersion ) {
//wait for unblock to happen before counting down invocation latches
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 16b3ff9..92312d2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -101,6 +101,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
public EntityIndexBatch index( final IndexScope indexScope, final Entity entity ) {
IndexValidationUtils.validateIndexScope( indexScope );
ValidationUtils.verifyEntityWrite( entity );
+ ValidationUtils.verifyVersion( entity.getVersion() );
final String context = createContextName(indexScope);
@@ -161,7 +162,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
String[] indexes = entityIndex.getIndexes(AliasedEntityIndex.AliasType.Read);
//get the default index if no alias exists yet
if(indexes == null ||indexes.length == 0){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 3a11ec7..16f4b3f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -113,9 +113,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer addWriteAliasTimer;
private final Timer addReadAliasTimer;
private final Timer searchTimer;
- private final Timer allVersionsTimerFuture;
- private final Timer deletePreviousTimerFuture;
- private final Meter errorMeter;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -190,16 +187,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer( EsEntityIndexImpl.class, "search.cursor.timer" );
this.getVersionsTimer =metricsFactory
.getTimer( EsEntityIndexImpl.class, "get.versions.timer" );
- this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer" );
- this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer" );
- this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.all.versions.timer.future" );
- this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "delete.previous.versions.timer.future" );
-
- this.errorMeter = metricsFactory.getMeter(EsEntityIndexImpl.class,"errors");
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f20dfd3c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 5259a26..d55073a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -273,7 +273,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
public void call( final BulkRequestBuilder bulkRequestBuilder ) {
sendRequest( bulkRequestBuilder );
}
- } ).toBlocking().last();
+ } ).toBlocking().lastOrDefault(null);
//call back all futures
Observable.from(operationMessages)
[16/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/001f3f85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/001f3f85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/001f3f85
Branch: refs/heads/USERGRID-460
Commit: 001f3f85d7b943b1ddb58f8191e97e16f92f330c
Parents: 67fa78e 6263594
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:45:29 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:45:29 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 10 ----
.../batch/service/JobSchedulerService.java | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 4 --
.../corepersistence/CpRelationManager.java | 14 -----
.../persistence/EntityManagerFactory.java | 2 -
.../cassandra/EntityManagerFactoryImpl.java | 3 -
.../cassandra/EntityManagerImpl.java | 23 +-------
.../cassandra/RelationManagerImpl.java | 58 +++-----------------
.../cassandra/index/ConnectedIndexScanner.java | 2 -
.../cassandra/index/IndexBucketScanner.java | 2 -
stack/pom.xml | 43 ---------------
stack/rest/pom.xml | 4 --
.../org/apache/usergrid/rest/RootResource.java | 6 --
stack/test-utils/pom.xml | 5 --
15 files changed, 10 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
[40/50] incubator-usergrid git commit: Updated futures impl for
different queues
Posted by gr...@apache.org.
Updated futures impl for different queues
Added onError to catch errors
Added a gauge so we can track index operation in flight
Updated queue scope to only be a name, since we only use them at the system level.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2d6ae369
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2d6ae369
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2d6ae369
Branch: refs/heads/USERGRID-460
Commit: 2d6ae3698e2faff733e73d17aff569a3850a6485
Parents: 361060e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 13 16:19:40 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 13 16:19:40 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 +-
.../corepersistence/StaleIndexCleanupTest.java | 4 +
.../persistence/core/future/BetterFuture.java | 43 ++++--
.../core/metrics/MetricsFactory.java | 9 ++
.../core/metrics/MetricsFactoryImpl.java | 121 ++++++++++------
stack/corepersistence/queryindex/pom.xml | 8 +-
.../persistence/index/IndexBufferConsumer.java | 11 ++
.../usergrid/persistence/index/IndexFig.java | 11 ++
.../index/IndexOperationMessage.java | 7 +-
.../persistence/index/guice/IndexModule.java | 11 +-
.../persistence/index/impl/BufferQueue.java | 22 ++-
.../index/impl/BufferQueueInMemoryImpl.java | 53 ++++---
.../index/impl/BufferQueueSQSImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 142 ++-----------------
.../index/impl/EsIndexBufferConsumerImpl.java | 131 ++++++++++-------
.../index/guice/TestIndexModule.java | 8 +-
.../index/impl/BufferQueueSQSImplTest.java | 5 +
.../impl/EntityConnectionIndexImplTest.java | 3 -
.../persistence/index/impl/EntityIndexTest.java | 1 -
.../persistence/index/impl/EsTestUtils.java | 48 -------
.../cassandra/ManagementServiceImpl.java | 9 +-
.../notifications/NotificationsService.java | 50 ++++---
22 files changed, 359 insertions(+), 353 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 3230faa..161037b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -71,12 +71,7 @@ public class CoreModule extends AbstractModule {
install( new CommonModule());
install(new CollectionModule());
install(new GraphModule());
- install( new IndexModule() {
- @Override
- public void wireBufferQueue() {
- bind(BufferQueue.class).to( BufferQueueSQSImpl.class );
- }
- } );
+ install( new IndexModule() );
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
// install(new QueueModule());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 82d8f93..8d31b69 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -310,6 +310,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
em.delete( thing );
}
+
+ //put this into the top of the queue, once it's acked we've been flushed
+ em.refreshIndex();
+
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
index 201fa9a..777ac52 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/future/BetterFuture.java
@@ -16,28 +16,53 @@
*/
package org.apache.usergrid.persistence.core.future;
+
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
+
/**
* Future without the exception nastiness
*/
-public class BetterFuture<T> extends FutureTask<T> {
- public BetterFuture(Callable<T> callable){
- super(callable);
+public class BetterFuture<T> extends FutureTask<T> {
+
+ private Throwable error;
+
+
+ public BetterFuture( Callable<T> callable ) {
+ super( callable );
}
- public void done(){
+
+ public void setError( final Throwable t ) {
+ this.error = t;
+ }
+
+
+ public void done() {
run();
}
- public T get(){
+
+ public T get() {
+
+ T returnValue = null;
+
try {
- return super.get();
- }catch (Exception e){
- throw new RuntimeException(e);
+ returnValue = super.get();
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+ catch ( ExecutionException e ) {
+ //swallow
}
- }
+ if ( error != null ) {
+ throw new RuntimeException( "Error in getting future", error );
+ }
+ return returnValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
index 453e556..62a5cb9 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
@@ -31,4 +31,13 @@ public interface MetricsFactory {
Counter getCounter(Class<?> klass, String name);
Meter getMeter(Class<?> klass, String name);
+
+ /**
+ * Get a gauge and create it
+ * @param clazz
+ * @param name
+ * @param gauge
+ * @return
+ */
+ void addGauge( Class<?> clazz, String name, Gauge<?> gauge );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 6d0881b..904e56a 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -17,17 +17,27 @@
package org.apache.usergrid.persistence.core.metrics;
-import com.codahale.metrics.*;
+import java.net.InetSocketAddress;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.net.InetSocketAddress;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
/**
* Singleton class to manage metrics.
@@ -39,74 +49,101 @@ public class MetricsFactoryImpl implements MetricsFactory {
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
- private ConcurrentHashMap<String,Metric> hashMap;
- private static final Logger LOG = LoggerFactory.getLogger(MetricsFactoryImpl.class);
+ private ConcurrentHashMap<String, Metric> hashMap;
+ private static final Logger LOG = LoggerFactory.getLogger( MetricsFactoryImpl.class );
+
@Inject
- public MetricsFactoryImpl(MetricsFig metricsFig) {
+ public MetricsFactoryImpl( MetricsFig metricsFig ) {
registry = new MetricRegistry();
String metricsHost = metricsFig.getHost();
- if(!metricsHost.equals("false")) {
- Graphite graphite = new Graphite(new InetSocketAddress(metricsHost, 2003));
- graphiteReporter = GraphiteReporter.forRegistry(registry)
- .prefixedWith("usergrid-metrics")
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .filter(MetricFilter.ALL)
- .build(graphite);
- graphiteReporter.start(30, TimeUnit.SECONDS);
- }else {
- LOG.warn("MetricsService:Logger not started.");
+ if ( !metricsHost.equals( "false" ) ) {
+ Graphite graphite = new Graphite( new InetSocketAddress( metricsHost, 2003 ) );
+ graphiteReporter = GraphiteReporter.forRegistry( registry ).prefixedWith( "usergrid-metrics" )
+ .convertRatesTo( TimeUnit.SECONDS )
+ .convertDurationsTo( TimeUnit.MILLISECONDS ).filter( MetricFilter.ALL )
+ .build( graphite );
+ graphiteReporter.start( 30, TimeUnit.SECONDS );
+ }
+ else {
+ LOG.warn( "MetricsService:Logger not started." );
}
hashMap = new ConcurrentHashMap<String, Metric>();
- jmxReporter = JmxReporter.forRegistry(registry).build();
+ jmxReporter = JmxReporter.forRegistry( registry ).build();
jmxReporter.start();
}
+
@Override
public MetricRegistry getRegistry() {
return registry;
}
+
@Override
- public Timer getTimer(Class<?> klass, String name) {
- return getMetric(Timer.class, klass, name);
+ public Timer getTimer( Class<?> klass, String name ) {
+ return getMetric( Timer.class, klass, name );
}
+
@Override
- public Histogram getHistogram(Class<?> klass, String name) {
- return getMetric(Histogram.class, klass, name);
+ public Histogram getHistogram( Class<?> klass, String name ) {
+ return getMetric( Histogram.class, klass, name );
}
+
@Override
- public Counter getCounter(Class<?> klass, String name) {
- return getMetric(Counter.class, klass, name);
+ public Counter getCounter( Class<?> klass, String name ) {
+ return getMetric( Counter.class, klass, name );
}
+
@Override
- public Meter getMeter(Class<?> klass, String name) {
- return getMetric(Meter.class, klass, name);
+ public Meter getMeter( Class<?> klass, String name ) {
+ return getMetric( Meter.class, klass, name );
}
- private <T> T getMetric(Class<T> metricClass, Class<?> klass, String name) {
+
+ @Override
+ public void addGauge( final Class<?> clazz, final String name, final Gauge<?> gauge ) {
+
+ this.getRegistry().register( MetricRegistry.name( clazz, name ), gauge );
+ }
+
+
+ private <T> T getMetric( Class<T> metricClass, Class<?> klass, String name ) {
String key = metricClass.getName() + klass.getName() + name;
- Metric metric = hashMap.get(key);
- if (metric == null) {
- if (metricClass == Histogram.class) {
- metric = this.getRegistry().histogram(MetricRegistry.name(klass, name));
+ Metric metric = hashMap.get( key );
+ if ( metric == null ) {
+ if ( metricClass == Histogram.class ) {
+ metric = this.getRegistry().histogram( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Timer.class) {
- metric = this.getRegistry().timer(MetricRegistry.name(klass, name));
+ if ( metricClass == Timer.class ) {
+ metric = this.getRegistry().timer( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Meter.class) {
- metric = this.getRegistry().meter(MetricRegistry.name(klass, name));
+ if ( metricClass == Meter.class ) {
+ metric = this.getRegistry().meter( MetricRegistry.name( klass, name ) );
}
- if (metricClass == Counter.class) {
- metric = this.getRegistry().counter(MetricRegistry.name(klass, name));
+ if ( metricClass == Counter.class ) {
+ metric = this.getRegistry().counter( MetricRegistry.name( klass, name ) );
}
- hashMap.put(key, metric);
+
+
+ hashMap.put( key, metric );
}
- return (T) metric;
+ return ( T ) metric;
+ }
+
+
+ /**
+ *
+ * @param metricClass
+ * @param klass
+ * @param name
+ * @return
+ */
+ private String getKey( Class<?> metricClass, Class<?> klass, String name ) {
+ return metricClass.getName() + klass.getName() + name;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index af843ad..5f01ee7 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -139,7 +139,13 @@
<classifier>tests</classifier>
</dependency>
-
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
index ac7489c..40c7852 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
@@ -23,4 +23,15 @@ package org.apache.usergrid.persistence.index;
* Classy class class.
*/
public interface IndexBufferConsumer {
+
+
+ /**
+ * Start the consumer
+ */
+ public void start();
+
+ /**
+ * Stop the consumers
+ */
+ public void stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 445789f..6be8234 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -24,6 +24,8 @@ import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
import org.safehaus.guicyfig.Key;
+import org.apache.usergrid.persistence.index.guice.QueueProvider;
+
@FigSingleton
public interface IndexFig extends GuicyFig {
@@ -86,6 +88,11 @@ public interface IndexFig extends GuicyFig {
*/
public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@Default( "127.0.0.1" )
@@ -190,4 +197,8 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
+ @Default( "LOCAL" )
+ @Key( ELASTICSEARCH_QUEUE_IMPL )
+ String getQueueImplementation();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index 33b68cd..5686e26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -128,6 +128,11 @@ public class IndexOperationMessage implements Serializable {
}
public void done() {
- getFuture().done();
+ //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
+ final BetterFuture<IndexOperationMessage> future = getFuture();
+
+ if(future != null ){
+ future.done();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index b03e1c0..95f3bd4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -23,6 +23,7 @@ import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
@@ -33,7 +34,7 @@ import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.safehaus.guicyfig.GuicyFigModule;
-public abstract class IndexModule extends AbstractModule {
+public class IndexModule extends AbstractModule {
@Override
protected void configure() {
@@ -50,14 +51,10 @@ public abstract class IndexModule extends AbstractModule {
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
- wireBufferQueue();
- }
+ bind( BufferQueue.class).toProvider( QueueProvider.class );
+ }
- /**
- * Write the <class>BufferQueue</class> for this implementation
- */
- public abstract void wireBufferQueue();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
index ffc3b90..76b49c2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
@@ -32,25 +32,37 @@ import org.apache.usergrid.persistence.index.IndexOperationMessage;
public interface BufferQueue {
/**
- * Offer the indexoperation message
+ * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
+ * Other queues may return the future after ack on the offer. See the implementation documentation for details.
* @param operation
*/
public void offer(final IndexOperationMessage operation);
/**
- * Perform a take, potentially blocking. Until takesize is available, or timeout has elapsed
+ * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
+ * May return less than the take size, but will never return null
+ *
* @param takeSize
* @param timeout
* @param timeUnit
- * @return
+ * @return A null safe lid
*/
public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
/**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented
+ * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
+ * This will set the future as done in in memory operations
+ *
* @param messages
*/
- public void ack(List<IndexOperationMessage> messages);
+ public void ack(final List<IndexOperationMessage> messages);
+
+ /**
+ * Mark these message as failed. Set the exception in the future on local operation
+ *
+ * @param messages
+ */
+ public void fail(final List<IndexOperationMessage> messages, final Throwable t);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index 6716fd1..998c086 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
@@ -47,7 +48,6 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
messages.offer( operation );
- operation.done();
}
@@ -55,30 +55,30 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
+ try {
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
- //loop until we're we're full or we time out
- do {
- try {
+ messages.drainTo( response, takeSize );
- final long remaining = endTime - System.currentTimeMillis();
+ //we got something, go process it
+ if ( response.size() > 0 ) {
+ return response;
+ }
- //we received 1, try to drain
- IndexOperationMessage polled = messages.poll( remaining, timeUnit );
- //drain
- if ( polled != null ) {
- response.add( polled );
- messages.drainTo( response, takeSize - response.size() );
- }
- }
- catch ( InterruptedException ie ) {
- //swallow
+ final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+ if ( polled != null ) {
+ response.add( polled );
+
+ //try to add more
+ messages.drainTo( response, takeSize - 1 );
}
}
- while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+
return response;
}
@@ -86,6 +86,23 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void ack( final List<IndexOperationMessage> messages ) {
- //no op for this
+ //if we have a future ack it
+ for ( final IndexOperationMessage op : messages ) {
+ op.done();
+ }
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+
+
+ for ( final IndexOperationMessage op : messages ) {
+ final BetterFuture<IndexOperationMessage> future = op.getFuture();
+
+ if ( future != null ) {
+ future.setError( t );
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 4a07704..b8d162b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -101,7 +101,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig,
final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
final QueueScope queueScope =
- new QueueScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), QUEUE_NAME );
+ new QueueScopeImpl( QUEUE_NAME );
this.queue = queueManagerFactory.getQueueManager( queueScope );
this.indexFig = indexFig;
@@ -260,6 +260,12 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+ //no op, just let it retry after the queue timeout
+ }
+
+
/** Read the object from Base64 string. */
private IndexOperationMessage fromString( String s ) throws IOException {
IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fa50734..8be044f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -47,6 +47,8 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
@@ -111,8 +113,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer addWriteAliasTimer;
private final Timer addReadAliasTimer;
private final Timer searchTimer;
- private final Timer allVersionsTimerFuture;
- private final Timer deletePreviousTimerFuture;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -127,7 +127,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexFig config;
- private final MetricsFactory metricsFactory;
//number of times to wait for the index to refresh properly.
@@ -148,8 +147,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private Timer refreshTimer;
private Timer cursorTimer;
private Timer getVersionsTimer;
- private Timer allVersionsTimer;
- private Timer deletePreviousTimer;
private final MapManager mapManager;
@@ -168,11 +165,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier( config, appScope );
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
- this.metricsFactory = metricsFactory;
this.addTimer = metricsFactory
.getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
this.removeAliasTimer = metricsFactory
@@ -191,14 +187,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
this.getVersionsTimer =metricsFactory
.getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
- this.allVersionsTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
- this.deletePreviousTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
- this.allVersionsTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer.future" );
- this.deletePreviousTimerFuture = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer.future" );
+
final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
@@ -394,8 +383,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
*/
private void createMappings(final String indexName) throws IOException {
- XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), DEFAULT_TYPE );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(),
+ DEFAULT_TYPE );
//Added For Graphite Metrics
@@ -421,7 +410,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName(indexScope);
+ final String context = IndexingUtils.createContextName( indexScope );
final String[] entityTypes = searchTypes.getTypeNames();
QueryBuilder qb = query.createQueryBuilder(context);
@@ -632,7 +621,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public int getPendingTasks() {
final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin()
- .cluster().pendingClusterTasks(new PendingClusterTasksRequest()).actionGet();
+ .cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
return tasksResponse.pendingTasks().size();
}
@@ -674,114 +663,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
-// @Override
-// public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
-// String idString = IndexingUtils.idString(entityId).toLowerCase();
-//
-// final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
-//
-// //Added For Graphite Metrics
-// final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
-// final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
-//
-// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
-// .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
-//
-// response.addListener( new ActionListener<DeleteByQueryResponse>() {
-//
-// @Override
-// public void onResponse( DeleteByQueryResponse response) {
-// timeDeleteAllVersions.stop();
-// logger
-// .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
-// entityId.getUuid(), response.status().toString() );
-//
-// checkDeleteByQueryResponse(tqb, response);
-// }
-//
-//
-// @Override
-// public void onFailure( Throwable e ) {
-// timeDeleteAllVersions.stop();
-// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
-// entityId.getUuid(), e);
-//
-//
-// }
-// });
-// timeDeleteAllVersionsFuture.stop();
-// return response;
-// }
-//
-//
-// @Override
-// public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
-//
-// String idString = IndexingUtils.idString( entityId ).toLowerCase();
-//
-// final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
-// QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
-// FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
-// );
-//
-// //Added For Graphite Metrics
-// //Checks the time from the execute to the response below
-// final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
-// final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
-// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
-// .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
-//
-// //Added For Graphite Metrics
-// response.addListener(new ActionListener<DeleteByQueryResponse>() {
-// @Override
-// public void onResponse(DeleteByQueryResponse response) {
-// timeDeletePreviousVersions.stop();
-// //error message needs to be retooled so that it describes the entity more throughly
-// logger
-// .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
-// entityId.getType(), entityId.getUuid(), version, response.status().toString());
-//
-// checkDeleteByQueryResponse( fqb, response );
-// }
-//
-//
-// @Override
-// public void onFailure( Throwable e ) {
-// timeDeletePreviousVersions.stop();
-// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
-// entityId.getUuid(), e );
-// }
-// } );
-//
-// timeDeletePreviousVersionFuture.stop();
-//
-// return response;
-// }
-
-
- /**
- * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
- */
- private void checkDeleteByQueryResponse(
- final QueryBuilder query, final DeleteByQueryResponse response ) {
-
- for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
- final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
-
- for ( ShardOperationFailedException failedException : failures ) {
- logger.error( String.format("Unable to delete by query %s. "
- + "Failed with code %d and reason %s on shard %s in index %s",
- query.toString(),
- failedException.status().getStatus(),
- failedException.reason(),
- failedException.shardId(),
- failedException.index() )
- );
- }
-
- }
- }
-
/**
* Completely delete an index.
@@ -856,8 +737,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public Health getIndexHealth() {
try {
- ClusterHealthResponse chr = esProvider.getClient().admin().cluster().health(
- new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)})).get();
+ final ActionFuture<ClusterHealthResponse> future = esProvider.getClient().admin().cluster().health(
+ new ClusterHealthRequest( new String[] { indexIdentifier.getIndex( null ) } ) );
+
+ //only wait 2 seconds max
+ ClusterHealthResponse chr = future.actionGet( 2000 );
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 836ec3d..5259a26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -20,31 +20,29 @@
package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexBufferConsumer;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
-import org.elasticsearch.action.ActionRequestBuilder;
+
import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
-import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
import rx.functions.Action1;
-import rx.functions.Action2;
import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
@@ -52,7 +50,6 @@ import rx.schedulers.Schedulers;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -69,6 +66,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Timer flushTimer;
private final Counter indexSizeCounter;
+ private final Counter indexErrorCounter;
private final Meter flushMeter;
private final Timer produceTimer;
private final BufferQueue bufferQueue;
@@ -80,13 +78,28 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private Object mutex = new Object();
+
+ private AtomicLong inFlight = new AtomicLong( );
+
@Inject
public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
- this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
- this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
+ this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "buffer.flush");
+ this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "buffer.meter");
+ this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "buffer.size");
+ this.indexErrorCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "error.count");
+
+ //wire up the gauge of inflight messages
+ metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return inFlight.longValue();
+ }
+ } );
+
+
+
this.config = config;
this.failureMonitor = new FailureMonitorImpl(config,provider);
this.client = provider.getClient();
@@ -130,66 +143,67 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private void startWorker(){
synchronized ( mutex) {
- final AtomicInteger countFail = new AtomicInteger();
+ Observable<List<IndexOperationMessage>> consumer = Observable.create(
+ new Observable.OnSubscribe<List<IndexOperationMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
- Observable<List<IndexOperationMessage>> consumer = Observable.create( new Observable.OnSubscribe<List<IndexOperationMessage>>() {
- @Override
- public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
+ //name our thread so it's easy to see
+ Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
- //name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
+ List<IndexOperationMessage> drainList;
+ do {
+ try {
- List<IndexOperationMessage> drainList;
- do {
- try {
+ Timer.Context timer = produceTimer.time();
- Timer.Context timer = produceTimer.time();
+ drainList = bufferQueue
+ .take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
+ TimeUnit.MILLISECONDS );
- drainList = bufferQueue.take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
- TimeUnit.MILLISECONDS );
+ subscriber.onNext( drainList );
- subscriber.onNext( drainList );
+ //take since we're in flight
+ inFlight.addAndGet( drainList.size() );
- timer.stop();
+ timer.stop();
+ }
- countFail.set( 0 );
- }
- catch ( EsRejectedExecutionException err ) {
- countFail.incrementAndGet();
- log.error(
- "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. " + "Failed {} consecutive times", config.getFailRefreshCount(),
- countFail.get() );
+ catch ( Exception e ) {
+ final long sleepTime = config.getFailureRetryTime();
- //es rejected the exception, sleep and retry in the queue
- try {
- Thread.sleep( config.getFailureRetryTime() );
- }
- catch ( InterruptedException e ) {
- //swallow
- }
- }
- catch ( Exception e ) {
- int count = countFail.incrementAndGet();
- log.error( "failed to dequeue", e );
- if ( count > 200 ) {
- log.error( "Shutting down index drain due to repetitive failures" );
+ log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, e );
+
+ try {
+ Thread.sleep( sleepTime );
+ }
+ catch ( InterruptedException ie ) {
+ //swallow
+ }
+
+ indexErrorCounter.inc();
}
}
+ while ( true );
}
- while ( true );
- }
- } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
+ } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
public void call( List<IndexOperationMessage> containerList ) {
- if ( containerList.size() > 0 ) {
- flushMeter.mark( containerList.size() );
- Timer.Context time = flushTimer.time();
- execute( containerList );
- time.stop();
+ if ( containerList.size() == 0 ) {
+ return;
}
+
+ flushMeter.mark( containerList.size() );
+ Timer.Context time = flushTimer.time();
+
+
+ execute( containerList );
+
+ time.stop();
+
}
} )
//ack after we process
@@ -197,6 +211,16 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
@Override
public void call( final List<IndexOperationMessage> indexOperationMessages ) {
bufferQueue.ack( indexOperationMessages );
+ //release so we know we've done processing
+ inFlight.addAndGet( -1 * indexOperationMessages.size() );
+ }
+ } ).doOnError( new Action1<Throwable>() {
+ @Override
+ public void call( final Throwable throwable ) {
+
+ log.error( "An exception occurred when trying to deque and write to elasticsearch. Ignoring",
+ throwable );
+ indexErrorCounter.inc();
}
} );
@@ -236,7 +260,8 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//collection all the operations into a single stream
.reduce( initRequest(), new Func2<BulkRequestBuilder, BatchRequest, BulkRequestBuilder>() {
@Override
- public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder,
+ final BatchRequest batchRequest ) {
batchRequest.doOperation( client, bulkRequestBuilder );
return bulkRequestBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 57c2fab..50b994d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -35,12 +35,6 @@ public class TestIndexModule extends TestModule {
// configure collections and our core astyanax framework
install( new CollectionModule() );
- install( new IndexModule() {
- @Override
- public void wireBufferQueue() {
- bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
-// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
- }
- } );
+ install( new IndexModule() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index 6922c15..9a362cb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
@@ -59,6 +60,10 @@ public class BufferQueueSQSImplTest {
@Rule
public MigrationManagerRule migrationManagerRule;
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
@Inject
public QueueManagerFactory queueManagerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index c5f3488..a399809 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -140,7 +140,6 @@ public class EntityConnectionIndexImplTest extends BaseIT {
personLikesIndex.refresh();
- EsTestUtils.waitForTasks(personLikesIndex);
Thread.sleep( 2000 );
// now, let's search for muffins
@@ -270,8 +269,6 @@ public class EntityConnectionIndexImplTest extends BaseIT {
batch.execute().get();
personLikesIndex.refresh();
- EsTestUtils.waitForTasks( personLikesIndex );
- Thread.sleep( 2000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex.search( searchScope,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a2135a3..ca9bf79 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -94,7 +94,6 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
- Thread.sleep(100000000);
testQueries( indexScope, searchTypes, entityIndex );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
deleted file mode 100644
index 30f0ed0..0000000
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsTestUtils.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import org.apache.usergrid.persistence.index.EntityIndex;
-
-
-/**
- * Utilities to make testing ES easier
- */
-public class EsTestUtils {
-
-
- /**
- * Checks to see if we have pending tasks in the cluster. If so waits until they are finished. Adding
- * new types can cause lag even after refresh since the type mapping needs applied
- * @param index
- */
- public static void waitForTasks(final EntityIndex index){
-
- while(index.getPendingTasks() > 0){
- try {
- Thread.sleep( 100 );
- }
- catch ( InterruptedException e ) {
- //swallow
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 35ed091..854c3e0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -540,7 +540,6 @@ public class ManagementServiceImpl implements ManagementService {
em.addToCollection( organizationEntity, "users", new SimpleEntityRef( User.ENTITY_TYPE, user.getUuid() ) );
- em.refreshIndex();
writeUserToken( smf.getManagementAppId(), organizationEntity, encryptionService
.plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.ORGANIZATION ), user.getUuid(),
@@ -557,7 +556,7 @@ public class ManagementServiceImpl implements ManagementService {
startOrganizationActivationFlow( organization );
- em.refreshIndex();
+
return organization;
}
@@ -1649,7 +1648,7 @@ public class ManagementServiceImpl implements ManagementService {
properties.put( "appUuid", applicationId );
Entity appInfo = em.create( applicationId, APPLICATION_INFO, properties );
- em.refreshIndex();
+
writeUserToken( smf.getManagementAppId(), appInfo, encryptionService
.plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
@@ -1670,7 +1669,7 @@ public class ManagementServiceImpl implements ManagementService {
+ ")</a> created a new application named " + applicationName, null );
}
- em.refreshIndex();
+
return new ApplicationInfo( applicationId, appInfo.getName() );
}
@@ -2376,7 +2375,7 @@ public class ManagementServiceImpl implements ManagementService {
if ( sendEmail ) {
startOrganizationActivationFlow( organization );
}
- em.refreshIndex();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d6ae369/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index 5eed002..abd77ee 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -16,39 +16,56 @@
*/
package org.apache.usergrid.services.notifications;
-import java.util.*;
-import com.codahale.metrics.*;
-import com.codahale.metrics.Timer;
-import com.google.inject.Injector;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.mq.Message;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PathQuery;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.entities.Device;
import org.apache.usergrid.persistence.entities.Notification;
import org.apache.usergrid.persistence.entities.Notifier;
import org.apache.usergrid.persistence.entities.Receipt;
+import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-import org.apache.usergrid.services.*;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+import org.apache.usergrid.services.AbstractCollectionService;
+import org.apache.usergrid.services.ServiceAction;
+import org.apache.usergrid.services.ServiceContext;
+import org.apache.usergrid.services.ServiceInfo;
+import org.apache.usergrid.services.ServiceManagerFactory;
+import org.apache.usergrid.services.ServiceParameter;
+import org.apache.usergrid.services.ServicePayload;
+import org.apache.usergrid.services.ServiceRequest;
+import org.apache.usergrid.services.ServiceResults;
+import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.entities.Device;
-import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
-import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
-import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.google.inject.Injector;
import rx.Observable;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
+import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+
public class NotificationsService extends AbstractCollectionService {
@@ -90,9 +107,8 @@ public class NotificationsService extends AbstractCollectionService {
postMeter = metricsService.getMeter(NotificationsService.class, "requests");
postTimer = metricsService.getTimer(this.getClass(), "execution_rest");
JobScheduler jobScheduler = new JobScheduler(sm,em);
- String name = ApplicationQueueManagerImpl.getQueueNames(props);
- QueueScopeFactory queueScopeFactory = getApplicationContext().getBean( Injector.class ).getInstance(QueueScopeFactory.class);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), name);
+ String name = ApplicationQueueManagerImpl.getQueueNames( props );
+ QueueScope queueScope = new QueueScopeImpl( name );
queueManagerFactory = getApplicationContext().getBean( Injector.class ).getInstance(QueueManagerFactory.class);
QueueManager queueManager = TEST_QUEUE_MANAGER !=null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
notificationQueueManager = new ApplicationQueueManagerImpl(jobScheduler,em,queueManager,metricsService,props);
[45/50] incubator-usergrid git commit: Fixes issue with atomically
moving indexes
Posted by gr...@apache.org.
Fixes issue with atomically moving indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c16bb003
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c16bb003
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c16bb003
Branch: refs/heads/USERGRID-460
Commit: c16bb003931e273dd8d5e57e833c82dac1927fed
Parents: 0334119
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 10:05:55 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 10:05:55 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 83 +++++++++-----------
1 file changed, 35 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c16bb003/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 16f4b3f..cae9cb0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -56,6 +56,8 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -110,8 +112,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexBufferProducer indexBatchBufferProducer;
private final IndexFig indexFig;
private final Timer addTimer;
- private final Timer addWriteAliasTimer;
- private final Timer addReadAliasTimer;
+ private final Timer updateAliasTimer;
private final Timer searchTimer;
/**
@@ -142,7 +143,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
private EsIndexCache aliasCache;
- private Timer removeAliasTimer;
private Timer mappingTimer;
private Timer refreshTimer;
private Timer cursorTimer;
@@ -171,12 +171,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.aliasCache = indexCache;
this.addTimer = metricsFactory
.getTimer( EsEntityIndexImpl.class, "add.timer" );
- this.removeAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "remove.alias.timer" );
- this.addReadAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "add.read.alias.timer" );
- this.addWriteAliasTimer = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "add.write.alias.timer" );
+ this.updateAliasTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "update.alias.timer" );
this.mappingTimer = metricsFactory
.getTimer( EsEntityIndexImpl.class, "create.mapping.timer" );
this.refreshTimer = metricsFactory
@@ -261,60 +257,51 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public void addAlias(final String indexSuffix) {
+
+ final Timer.Context timeRemoveAlias = updateAliasTimer.time();
try {
- Boolean isAck;
- String indexName = indexIdentifier.getIndex(indexSuffix);
- final AdminClient adminClient = esProvider.getClient().admin();
- String[] indexNames = getIndexes(AliasType.Write);
- for ( String currentIndex : indexNames ) {
+ String indexName = indexIdentifier.getIndex(indexSuffix);
+ final AdminClient adminClient = esProvider.getClient().admin();
- final Timer.Context timeRemoveAlias = removeAliasTimer.time();
+ String[] indexNames = getIndexesFromEs( AliasType.Write );
- try {
- //Added For Graphite Metrics
- isAck = adminClient.indices().prepareAliases().removeAlias( currentIndex, alias.getWriteAlias() )
- .execute().actionGet().isAcknowledged();
+ final IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
- logger.info( "Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck );
- }
- catch ( AliasesMissingException aie ) {
- logger.info( "Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias,
- aie.getMessage() );
- continue;
- }
- catch ( InvalidAliasNameException iane ) {
- logger.info( "Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias,
- iane.getMessage() );
- continue;
- }
- finally {
- timeRemoveAlias.stop();
- }
+ //remove the write alias from it's target
+ for ( String currentIndex : indexNames ) {
+ aliasesRequestBuilder.removeAlias( currentIndex, alias.getWriteAlias() );
+ logger.info("Removing existing write Alias Name [{}] from Index [{}]", alias.getReadAlias(), currentIndex);
}
//Added For Graphite Metrics
- Timer.Context timeAddReadAlias = addReadAliasTimer.time();
+
// add read alias
- isAck = adminClient.indices().prepareAliases().addAlias(
- indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
- timeAddReadAlias.stop();
- logger.info("Created new read Alias Name [{}] ACK=[{}]", alias.getReadAlias(), isAck);
+ aliasesRequestBuilder.addAlias( indexName, alias.getReadAlias());
+ logger.info("Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName);
+
- //Added For Graphite Metrics
- Timer.Context timeAddWriteAlias = addWriteAliasTimer.time();
//add write alias
- isAck = adminClient.indices().prepareAliases().addAlias(
- indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- timeAddWriteAlias.stop();
- logger.info("Created new write Alias Name [{}] ACK=[{}]", alias.getWriteAlias(), isAck);
+ aliasesRequestBuilder.addAlias( indexName, alias.getWriteAlias() );
- aliasCache.invalidate(alias);
+ logger.info("Created new write Alias Name [{}] on Index [{}]", alias.getWriteAlias(), indexName);
+
+ final IndicesAliasesResponse result = aliasesRequestBuilder.execute().actionGet();
- } catch (Exception e) {
- logger.warn("Failed to create alias ", e);
+ final boolean isAcknowledged = result.isAcknowledged();
+
+ if(!isAcknowledged){
+ throw new RuntimeException( "Unable to add aliases to the new index " + indexSuffix );
+ }
+
+ }
+ finally{
+ //invalidate the alias
+ aliasCache.invalidate(alias);
+ //stop the timer
+ timeRemoveAlias.stop();
}
}
[05/50] incubator-usergrid git commit: Refactored to write to
cassandra for data, and to SQS for the id of the data.
Posted by gr...@apache.org.
Refactored to write to cassandra for data, and to SQS for the id of the data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8d8eb060
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8d8eb060
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8d8eb060
Branch: refs/heads/USERGRID-460
Commit: 8d8eb060e20bd9942d591227ae2cdd5337214b81
Parents: 9111d94
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 20:38:52 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 20:38:52 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/map/MapManager.java | 35 ++--
.../persistence/map/impl/MapManagerImpl.java | 8 +
.../persistence/map/impl/MapSerialization.java | 9 +
.../map/impl/MapSerializationImpl.java | 93 +++++++++
.../persistence/map/MapManagerTest.java | 49 ++++-
stack/corepersistence/queryindex/pom.xml | 1 -
.../index/IndexOperationMessage.java | 36 +++-
.../index/impl/BufferQueueSQSImpl.java | 167 ++++++++++++++--
.../persistence/index/impl/DeIndexRequest.java | 10 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 196 +++++++++++--------
.../index/guice/TestIndexModule.java | 4 +-
.../index/impl/BufferQueueSQSImplTest.java | 145 ++++++++++++++
.../impl/EntityConnectionIndexImplTest.java | 4 +-
.../persistence/index/impl/EntityIndexTest.java | 21 +-
.../queue/impl/SQSQueueManagerImpl.java | 32 ++-
15 files changed, 656 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
index 62fe57d..69e0874 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
@@ -1,23 +1,26 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.usergrid.persistence.map;
+import java.util.Collection;
+import java.util.Map;
import java.util.UUID;
@@ -33,6 +36,14 @@ public interface MapManager {
*/
public String getString( final String key );
+
+ /**
+ * Get the values for all the keys. If a value does not exist, it won't be present in the map
+ * @param keys
+ * @return
+ */
+ public Map<String, String> getStrings(final Collection<String> keys);
+
/**
* Return the string, null if not found
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java
index c077c7d..fb2e7ff 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerImpl.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.persistence.map.impl;
+import java.util.Collection;
+import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.persistence.map.MapManager;
@@ -51,6 +53,12 @@ public class MapManagerImpl implements MapManager {
@Override
+ public Map<String, String> getStrings( final Collection<String> keys ) {
+ return mapSerialization.getStrings( scope, keys );
+ }
+
+
+ @Override
public void putString( final String key, final String value ) {
mapSerialization.putString( scope, key, value );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerialization.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerialization.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerialization.java
index 6e7e328..2e958c2 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerialization.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerialization.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.persistence.map.impl;
+import java.util.Collection;
+import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
@@ -33,6 +35,13 @@ public interface MapSerialization extends Migration {
public String getString( final MapScope scope, final String key );
/**
+ * Get strings from the map
+ * @param keys
+ * @return
+ */
+ public Map<String, String> getStrings( final MapScope scope, final Collection<String> keys );
+
+ /**
* Return the string, null if not found
*/
public void putString( final MapScope scope, final String key, final String value );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index 715c202..825d636 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -18,9 +18,12 @@
*/
package org.apache.usergrid.persistence.map.impl;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import com.google.common.base.Preconditions;
@@ -50,6 +53,9 @@ import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.CompositeBuilder;
import com.netflix.astyanax.model.CompositeParser;
+import com.netflix.astyanax.model.Row;
+import com.netflix.astyanax.model.Rows;
+import com.netflix.astyanax.query.ColumnFamilyQuery;
import com.netflix.astyanax.serializers.BooleanSerializer;
import com.netflix.astyanax.serializers.StringSerializer;
@@ -73,6 +79,9 @@ public class MapSerializationImpl implements MapSerialization {
private static final StringSerializer STRING_SERIALIZER = StringSerializer.get();
+ private static final StringResultsBuilder STRING_RESULTS_BUILDER = new StringResultsBuilder();
+
+
/**
* CFs where the row key contains the source node id
*/
@@ -126,6 +135,12 @@ public class MapSerializationImpl implements MapSerialization {
@Override
+ public Map<String, String> getStrings(final MapScope scope, final Collection<String> keys ) {
+ return getValues( scope, keys, STRING_RESULTS_BUILDER );
+ }
+
+
+ @Override
public void putString( final MapScope scope, final String key, final String value ) {
final RowOp op = new RowOp() {
@Override
@@ -371,6 +386,49 @@ public class MapSerializationImpl implements MapSerialization {
}
+ /**
+ * Get multiple values, using the string builder
+ * @param scope
+ * @param keys
+ * @param builder
+ * @param <T>
+ * @return
+ */
+ private <T> T getValues(final MapScope scope, final Collection<String> keys, final ResultsBuilder<T> builder) {
+
+
+ final List<ScopedRowKey<MapEntryKey>> rowKeys = new ArrayList<>( keys.size() );
+
+ for(final String key: keys){
+ //add it to the entry
+ final ScopedRowKey<MapEntryKey> entryRowKey = MapEntryKey.fromKey(scope, key);
+
+ rowKeys.add( entryRowKey );
+
+ }
+
+
+
+ //now get all columns, including the "old row key value"
+ try {
+ final Rows<ScopedRowKey<MapEntryKey>, Boolean>
+ rows = keyspace.prepareQuery( MAP_ENTRIES ).getKeySlice( rowKeys ).withColumnSlice( true )
+ .execute().getResult();
+
+
+ return builder.buildResults( rows );
+ }
+ catch ( NotFoundException nfe ) {
+ //nothing to return
+ return null;
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to cassandra", e );
+ }
+ }
+
+
+
private void executeBatch(MutationBatch batch) {
try {
batch.execute();
@@ -449,4 +507,39 @@ public class MapSerializationImpl implements MapSerialization {
return ScopedRowKey.fromKey( mapScope.getApplication(), new MapEntryKey( mapScope.getName(), key ) );
}
}
+
+
+ /**
+ * Build the results from the row keys
+ * @param <T>
+ */
+ private static interface ResultsBuilder<T> {
+
+ public T buildResults(final Rows<ScopedRowKey<MapEntryKey>, Boolean> rows);
+ }
+
+ public static class StringResultsBuilder implements ResultsBuilder<Map<String, String>>{
+
+ @Override
+ public Map<String, String> buildResults( final Rows<ScopedRowKey<MapEntryKey>, Boolean> rows ) {
+ final int size = rows.size();
+
+ final Map<String, String> results = new HashMap<>(size);
+
+ for(int i = 0; i < size; i ++){
+
+ final Row<ScopedRowKey<MapEntryKey>, Boolean> row = rows.getRowByIndex( i );
+
+ final String value = row.getColumns().getStringValue( true, null );
+
+ if(value == null){
+ continue;
+ }
+
+ results.put( row.getKey().getKey().key, value );
+ }
+
+ return results;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
index df4394e..41286ab 100644
--- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
+++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
@@ -20,6 +20,9 @@
package org.apache.usergrid.persistence.map;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -34,9 +37,11 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.map.guice.TestMapModule;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
+import static junit.framework.TestCase.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@@ -79,6 +84,47 @@ public class MapManagerTest {
@Test
+ public void multiReadNoKey() {
+ MapManager mm = mmf.createMapManager( this.scope );
+
+ final String key = UUIDGenerator.newTimeUUID().toString();
+
+ final Map<String, String> results = mm.getStrings( Collections.singleton( key ) );
+
+ assertNotNull( results );
+
+ final String shouldBeMissing = results.get( key );
+
+ assertNull( shouldBeMissing );
+ }
+
+
+ @Test
+ public void writeReadStringBatch() {
+ MapManager mm = mmf.createMapManager( this.scope );
+
+ final String key1 = "key1";
+ final String value1 = "value1";
+
+ mm.putString( key1, value1 );
+
+
+ final String key2 = "key2";
+ final String value2 = "value2";
+
+ mm.putString( key2, value2 );
+
+
+ final Map<String, String> returned = mm.getStrings( Arrays.asList( key1, key2 ) );
+
+ assertNotNull( returned );
+
+ assertEquals( value1, returned.get( key1 ) );
+ assertEquals( value2, returned.get( key2 ) );
+ }
+
+
+ @Test
public void writeReadStringTTL() throws InterruptedException {
MapManager mm = mmf.createMapManager( this.scope );
@@ -106,8 +152,7 @@ public class MapManagerTest {
//now read it should be gone
final String timedOut = mm.getString( key );
- assertNull("Value was not returned", timedOut);
-
+ assertNull( "Value was not returned", timedOut );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index a5fbf6a..af843ad 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -145,7 +145,6 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
- <scope>test</scope>
</dependency>
<!-- common stuff, logging, etc.-->
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index 7d8a859..a7388d6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -60,17 +60,17 @@ public class IndexOperationMessage implements Serializable {
public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
- indexRequests.addAll( indexRequests );
+ this.indexRequests.addAll( indexRequests );
}
public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
- deIndexRequests.add( deIndexRequest );
+ this.deIndexRequests.add( deIndexRequest );
}
public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
- deIndexRequests.addAll( deIndexRequests );
+ this.deIndexRequests.addAll( deIndexRequests );
}
@@ -96,4 +96,34 @@ public class IndexOperationMessage implements Serializable {
public BetterFuture<IndexOperationMessage> getFuture() {
return containerFuture;
}
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final IndexOperationMessage that = ( IndexOperationMessage ) o;
+
+ if ( !deIndexRequests.equals( that.deIndexRequests ) ) {
+ return false;
+ }
+ if ( !indexRequests.equals( that.indexRequests ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = indexRequests.hashCode();
+ result = 31 * result + deIndexRequests.hashCode();
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 833e045..c6acb36 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -23,56 +23,132 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.QueueMessage;
import org.apache.usergrid.persistence.queue.QueueScope;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+/**
+ * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
+ * performing
+ */
@Singleton
public class BufferQueueSQSImpl implements BufferQueue {
+ private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
+
/** Hacky, copied from CPEntityManager b/c we can't access it here */
public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
private static final String QUEUE_NAME = "es_queue";
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
private final QueueManager queue;
+ private final MapManager mapManager;
private final IndexFig indexFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
@Inject
- public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig ) {
- final QueueScope scope =
+ public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope =
new QueueScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), QUEUE_NAME );
- this.queue = queueManagerFactory.getQueueManager( scope );
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
this.indexFig = indexFig;
+
+ final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
+
+ this.mapManager = mapManagerFactory.createMapManager( scope );
+
+
+ this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
}
@Override
public void offer( final IndexOperationMessage operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
try {
- this.queue.sendMessage( operation );
+
+ final String payLoad = toString( operation );
+
+ //write to cassandra
+ this.mapManager.putString( identifier.toString(), payLoad, TTL );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
operation.getFuture().run();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
}
+ finally {
+ timer.stop();
+ }
}
@@ -83,23 +159,71 @@ public class BufferQueueSQSImpl implements BufferQueue {
final int actualTake = Math.min( 10, takeSize );
- List<QueueMessage> messages = queue
- .getMessages( actualTake, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- IndexOperationMessage.class );
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
- final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
- for ( final QueueMessage message : messages ) {
- final IndexOperationMessage messageBody = ( IndexOperationMessage ) message.getBody();
+ final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage(message, messageBody );
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
- response.add( operation );
- }
- return response;
+ if(messages.size() == 0){
+ return response;
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+ //look up the values
+ final Map<String, String> values = mapManager.getStrings( mapEntries );
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String key = message.getBody().toString();
+
+ //now see if the key was there
+ final String payload = values.get( key );
+
+ //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
+ // a DLQ
+
+ if ( payload == null ) {
+ continue;
+ }
+
+ final IndexOperationMessage messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
}
@@ -107,7 +231,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
public void ack( final List<IndexOperationMessage> messages ) {
//nothing to do
- if(messages.size() == 0){
+ if ( messages.size() == 0 ) {
return;
}
@@ -121,6 +245,19 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
+ /** Read the object from Base64 string. */
+ private IndexOperationMessage fromString( String s ) throws IOException {
+ IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexOperationMessage o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+
/**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index c63c4df..9f3ce66 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -37,9 +37,9 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class")
public class DeIndexRequest implements BatchRequest {
- public final String[] indexes;
- public final String entityType;
- public final String documentId;
+ public String[] indexes;
+ public String entityType;
+ public String documentId;
public DeIndexRequest( final String[] indexes, final String entityType, final String documentId) {
@@ -49,6 +49,10 @@ public class DeIndexRequest implements BatchRequest {
}
+ public DeIndexRequest() {
+ }
+
+
@Override
public void doOperation(final Client client, final BulkRequestBuilder bulkRequest ){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 8547889..3fc3e77 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -42,7 +42,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
+import rx.Subscription;
import rx.functions.Action1;
+import rx.functions.Action2;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -60,17 +62,23 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final IndexFig config;
private final FailureMonitorImpl failureMonitor;
private final Client client;
- private final Observable<List<IndexOperationMessage>> consumer;
+
private final Timer flushTimer;
private final Counter indexSizeCounter;
private final Meter flushMeter;
private final Timer produceTimer;
private final BufferQueue bufferQueue;
+ //the actively running subscription
+ private Subscription subscription;
+
+ private Observable<List<IndexOperationMessage>> consumer;
+
+ private Object mutex = new Object();
+
@Inject
public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider
provider, final MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
- this.bufferQueue = bufferQueue;
this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
@@ -78,81 +86,101 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
this.failureMonitor = new FailureMonitorImpl(config,provider);
this.client = provider.getClient();
this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
+ this.bufferQueue = bufferQueue;
+
- final AtomicInteger countFail = new AtomicInteger();
//batch up sets of some size and send them in batch
- this.consumer = Observable.create( new Observable.OnSubscribe<List<IndexOperationMessage>>() {
- @Override
- public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
+ start();
+ }
+
+
+ public void start() {
+ synchronized ( mutex) {
- //name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
+ final AtomicInteger countFail = new AtomicInteger();
- List<IndexOperationMessage> drainList;
- do {
- try {
+ Observable<List<IndexOperationMessage>> consumer = Observable.create( new Observable.OnSubscribe<List<IndexOperationMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
+ //name our thread so it's easy to see
+ Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
- Timer.Context timer = produceTimer.time();
+ List<IndexOperationMessage> drainList;
+ do {
+ try {
- drainList = bufferQueue
- .take( config.getIndexBufferSize(), config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS );
+ Timer.Context timer = produceTimer.time();
- subscriber.onNext( drainList );
+ drainList = bufferQueue.take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
+ TimeUnit.MILLISECONDS );
- timer.stop();
+ subscriber.onNext( drainList );
- countFail.set( 0 );
- }
- catch ( EsRejectedExecutionException err ) {
- countFail.incrementAndGet();
- log.error(
- "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. "
- + "Failed {} consecutive times",
- config.getFailRefreshCount(), countFail.get() );
-
- //es rejected the exception, sleep and retry in the queue
- try {
- Thread.sleep( config.getFailureRetryTime() );
+
+ timer.stop();
+
+ countFail.set( 0 );
}
- catch ( InterruptedException e ) {
- //swallow
+ catch ( EsRejectedExecutionException err ) {
+ countFail.incrementAndGet();
+ log.error(
+ "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. " + "Failed {} consecutive times", config.getFailRefreshCount(),
+ countFail.get() );
+
+ //es rejected the exception, sleep and retry in the queue
+ try {
+ Thread.sleep( config.getFailureRetryTime() );
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
}
- }
- catch ( Exception e ) {
- int count = countFail.incrementAndGet();
- log.error( "failed to dequeue", e );
- if ( count > 200 ) {
- log.error( "Shutting down index drain due to repetitive failures" );
+ catch ( Exception e ) {
+ int count = countFail.incrementAndGet();
+ log.error( "failed to dequeue", e );
+ if ( count > 200 ) {
+ log.error( "Shutting down index drain due to repetitive failures" );
+ }
}
}
+ while ( true );
}
- while ( true );
- }
- } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
- @Override
- public void call( List<IndexOperationMessage> containerList ) {
- if ( containerList.size() > 0 ) {
- flushMeter.mark( containerList.size() );
- Timer.Context time = flushTimer.time();
- execute( containerList );
- time.stop();
+ } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
+ @Override
+ public void call( List<IndexOperationMessage> containerList ) {
+ if ( containerList.size() > 0 ) {
+ flushMeter.mark( containerList.size() );
+ Timer.Context time = flushTimer.time();
+ execute( containerList );
+ time.stop();
+ }
}
+ } )
+ //ack after we process
+ .doOnNext( new Action1<List<IndexOperationMessage>>() {
+ @Override
+ public void call( final List<IndexOperationMessage> indexOperationMessages ) {
+ bufferQueue.ack( indexOperationMessages );
+ }
+ } );
+
+ //start in the background
+ subscription = consumer.subscribe();
+ }
+ }
+
+
+ public void stop() {
+ synchronized ( mutex ) {
+ //stop consuming
+ if(subscription != null) {
+ subscription.unsubscribe();
}
-} )
- //ack after we process
- .doOnNext( new Action1<List<IndexOperationMessage>>() {
- @Override
- public void call( final List<IndexOperationMessage> indexOperationMessages ) {
- bufferQueue.ack( indexOperationMessages );
- }
- } );
-
- //start in the background
- consumer.subscribe();
+ }
}
/**
@@ -165,43 +193,36 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
//process and flatten all the messages to builder requests
- Observable<BatchRequest> flattenMessages = Observable.from(operationMessages)
- .subscribeOn(Schedulers.io())
- .flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
- @Override
- public Observable<BatchRequest> call( IndexOperationMessage operationMessage ) {
- final Observable<DeIndexRequest> deIndex = Observable.from( operationMessage.getDeIndexRequests () );
- final Observable<IndexRequest> index = Observable.from( operationMessage.getIndexRequests() );
-
- return Observable.merge( deIndex, index );
- }
- } );
-
+ Observable<IndexOperationMessage> flattenMessages = Observable.from( operationMessages );
//batch shard operations into a bulk request
- flattenMessages.toList()
- .doOnNext(new Action1<List<BatchRequest>>() {
- @Override
- public void call(List<BatchRequest> builders) {
- try {
- final BulkRequestBuilder bulkRequest = initRequest();
- for (BatchRequest builder : builders) {
- indexSizeCounter.dec();
+ flattenMessages.flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
+ @Override
+ public Observable<BatchRequest> call( final IndexOperationMessage indexOperationMessage ) {
+ final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
+ final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
- builder.doOperation( client, bulkRequest );
- }
- sendRequest(bulkRequest);
- }catch (Exception e){
- log.error("Failed while sending bulk",e);
- }
- }
- })
- .toBlocking().lastOrDefault(null);
+ return Observable.merge( index, deIndex );
+ }
+ } )
+ //collection all the operations into a single stream
+ .collect( initRequest(), new Action2<BulkRequestBuilder, BatchRequest>() {
+ @Override
+ public void call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ batchRequest.doOperation( client, bulkRequestBuilder );
+ }
+ } )
+ //send the request off to ES
+ .doOnNext( new Action1<BulkRequestBuilder>() {
+ @Override
+ public void call( final BulkRequestBuilder bulkRequestBuilder ) {
+ sendRequest( bulkRequestBuilder );
+ }
+ } ).toBlocking().last();
//call back all futures
Observable.from(operationMessages)
- .subscribeOn(Schedulers.io())
.doOnNext(new Action1<IndexOperationMessage>() {
@Override
public void call(IndexOperationMessage operationMessage) {
@@ -211,6 +232,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.toBlocking().lastOrDefault(null);
}
+
/**
* initialize request
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 57c2fab..7d7a18d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -38,8 +38,8 @@ public class TestIndexModule extends TestModule {
install( new IndexModule() {
@Override
public void wireBufferQueue() {
- bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
-// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
+// bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
+ bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
new file mode 100644
index 0000000..4a4672e
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import static org.junit.Assert.*;
+
+
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+@NotThreadSafe
+public class BufferQueueSQSImplTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+ @Inject
+ private BufferQueueSQSImpl bufferQueueSQS;
+
+ @Inject
+ private EsIndexBufferConsumerImpl esIndexBufferConsumer;
+
+
+ @Before
+ public void stop() {
+ esIndexBufferConsumer.stop();
+ }
+
+
+ @After
+ public void after() {
+ esIndexBufferConsumer.start();
+ }
+
+
+
+
+ @Test
+ public void testMessageIndexing(){
+
+ final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
+ final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testType1", "testDoc1",request1Data );
+
+
+ final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
+ final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testType2", "testDoc2",request2Data );
+
+
+ //de-index request
+ final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, "testType3", "testId3" );
+
+ final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, "testType4", "testId4" );
+
+
+
+
+ IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
+ indexOperationMessage.addIndexRequest( indexRequest1);
+ indexOperationMessage.addIndexRequest( indexRequest2);
+
+ indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
+ indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
+
+ bufferQueueSQS.offer( indexOperationMessage );
+
+ //wait for it to send to SQS
+ indexOperationMessage.getFuture().get();
+
+ //now get it back
+
+ final List<IndexOperationMessage> ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+
+ assertTrue(ops.size() > 1);
+
+ final IndexOperationMessage returnedOperation = ops.get( 0 );
+
+ //get the operations out
+
+ final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
+
+ assertTrue(indexRequestSet.contains(indexRequest1));
+ assertTrue(indexRequestSet.contains(indexRequest2));
+
+
+ final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
+
+ assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
+ assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
+
+
+
+ //now ack the message
+
+ bufferQueueSQS.ack( ops );
+
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 215ff57..c5f3488 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -141,7 +141,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
EsTestUtils.waitForTasks(personLikesIndex);
- Thread.sleep( 30000 );
+ Thread.sleep( 2000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex
@@ -271,7 +271,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
personLikesIndex.refresh();
EsTestUtils.waitForTasks( personLikesIndex );
- Thread.sleep( 30000 );
+ Thread.sleep( 2000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex.search( searchScope,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a15053c..a2135a3 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -24,12 +24,14 @@ import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.model.field.ArrayField;
import org.apache.usergrid.persistence.model.field.EntityObjectField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
@@ -69,8 +71,14 @@ public class EntityIndexTest extends BaseIT {
@Inject
public EntityIndexFactory eif;
+ //TODO T.N. Remove this when we move the cursor mapping back to core
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
@Test
- public void testIndex() throws IOException {
+ public void testIndex() throws IOException, InterruptedException {
Id appId = new SimpleId( "application" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
@@ -86,7 +94,9 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ Thread.sleep(100000000);
+
+ testQueries( indexScope, searchTypes, entityIndex );
}
@Test
@@ -660,15 +670,12 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < size; i++ ) {
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "edanuff" );
- properties.put( "email", "ed@anuff.com" );
- properties.put( "middlename", middleName );
Map entityMap = new HashMap() {{
put( "username", "edanuff" );
put( "email", "ed@anuff.com" );
put( "middlename", middleName );
+ put( "created", System.nanoTime() );
}};
final Id userId = new SimpleId( "user" );
@@ -700,7 +707,7 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
- final Query query = Query.fromQL( "select *" );
+ final Query query = Query.fromQL( "select * order by created" );
query.setLimit( limit );
if ( cursor != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d8eb060/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index 10aa621..e2c5c1e 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -48,20 +48,19 @@ import java.util.concurrent.TimeUnit;
public class SQSQueueManagerImpl implements QueueManager {
private static final Logger LOG = LoggerFactory.getLogger(SQSQueueManagerImpl.class);
- private AmazonSQSClient sqs;
- private QueueScope scope;
- private QueueFig fig;
+ private final AmazonSQSClient sqs;
+ private final QueueScope scope;
private ObjectMapper mapper;
private static SmileFactory smileFactory = new SmileFactory();
- private static LoadingCache<SqsLoader, Queue> urlMap = CacheBuilder.newBuilder()
+ private LoadingCache<SqsLoader, Queue> urlMap = CacheBuilder.newBuilder()
.maximumSize(1000)
.build(new CacheLoader<SqsLoader, Queue>() {
@Override
public Queue load(SqsLoader queueLoader) throws Exception {
Queue queue = null;
try {
- GetQueueUrlResult result = queueLoader.getClient().getQueueUrl(queueLoader.getKey());
+ GetQueueUrlResult result = sqs.getQueueUrl(queueLoader.getKey());
queue = new Queue(result.getQueueUrl());
} catch (QueueDoesNotExistException queueDoesNotExistException) {
queue = null;
@@ -73,7 +72,7 @@ public class SQSQueueManagerImpl implements QueueManager {
String name = queueLoader.getKey();
CreateQueueRequest createQueueRequest = new CreateQueueRequest()
.withQueueName(name);
- CreateQueueResult result = queueLoader.getClient().createQueue(createQueueRequest);
+ CreateQueueResult result = sqs.createQueue(createQueueRequest);
String url = result.getQueueUrl();
queue = new Queue(url);
LOG.info("Created queue with url {}", url);
@@ -85,7 +84,6 @@ public class SQSQueueManagerImpl implements QueueManager {
@Inject
public SQSQueueManagerImpl(@Assisted QueueScope scope, QueueFig fig){
- this.fig = fig;
this.scope = scope;
try {
UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
@@ -99,8 +97,7 @@ public class SQSQueueManagerImpl implements QueueManager {
// mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
} catch ( Exception e ) {
- LOG.warn("failed to setup SQS",e);
-// throw new RuntimeException("Error setting up mapper", e);
+ throw new RuntimeException("Error setting up mapper", e);
}
}
@@ -127,14 +124,14 @@ public class SQSQueueManagerImpl implements QueueManager {
}
waitTime = waitTime/1000;
String url = getQueue().getUrl();
- LOG.info("Getting {} messages from {}", limit, url);
+ LOG.debug( "Getting {} messages from {}", limit, url);
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(url);
receiveMessageRequest.setMaxNumberOfMessages(limit);
receiveMessageRequest.setVisibilityTimeout(transactionTimeout);
receiveMessageRequest.setWaitTimeSeconds(waitTime);
ReceiveMessageResult result = sqs.receiveMessage(receiveMessageRequest);
List<Message> messages = result.getMessages();
- LOG.info("Received {} messages from {}",messages.size(),url);
+ LOG.debug( "Received {} messages from {}", messages.size(), url);
List<QueueMessage> queueMessages = new ArrayList<>(messages.size());
for (Message message : messages) {
Object body ;
@@ -157,7 +154,7 @@ public class SQSQueueManagerImpl implements QueueManager {
return;
}
String url = getQueue().getUrl();
- LOG.info("Sending Messages...{} to {}", bodies.size(), url);
+ LOG.debug( "Sending Messages...{} to {}", bodies.size(), url);
SendMessageBatchRequest request = new SendMessageBatchRequest(url);
List<SendMessageBatchRequestEntry> entries = new ArrayList<>(bodies.size());
@@ -180,7 +177,7 @@ public class SQSQueueManagerImpl implements QueueManager {
return;
}
String url = getQueue().getUrl();
- LOG.info("Sending Message...{} to {}",body.toString(),url);
+ LOG.debug( "Sending Message...{} to {}", body.toString(), url);
final String stringBody = toString(body);
@@ -192,7 +189,7 @@ public class SQSQueueManagerImpl implements QueueManager {
@Override
public void commitMessage(QueueMessage queueMessage) {
String url = getQueue().getUrl();
- LOG.info("Commit message {} to queue {}",queueMessage.getMessageId(),url);
+ LOG.debug( "Commit message {} to queue {}", queueMessage.getMessageId(), url);
sqs.deleteMessage(new DeleteMessageRequest()
.withQueueUrl(url)
@@ -203,7 +200,7 @@ public class SQSQueueManagerImpl implements QueueManager {
@Override
public void commitMessages(List<QueueMessage> queueMessages) {
String url = getQueue().getUrl();
- LOG.info("Commit messages {} to queue {}",queueMessages.size(),url);
+ LOG.debug( "Commit messages {} to queue {}", queueMessages.size(), url);
List<DeleteMessageBatchRequestEntry> entries = new ArrayList<>();
for(QueueMessage message : queueMessages){
entries.add(new DeleteMessageBatchRequestEntry(message.getMessageId(),message.getHandle()));
@@ -233,16 +230,11 @@ public class SQSQueueManagerImpl implements QueueManager {
public class SqsLoader {
private final String key;
- private final AmazonSQSClient client;
public SqsLoader(String key, AmazonSQSClient client) {
this.key = key;
- this.client = client;
}
- public AmazonSQSClient getClient() {
- return client;
- }
public String getKey() {
return key;
[02/50] incubator-usergrid git commit: Refactored index messages to
be serializable.
Posted by gr...@apache.org.
Refactored index messages to be serializable.
Consumers now rolls up index requests and flushes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/23919745
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/23919745
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/23919745
Branch: refs/heads/USERGRID-460
Commit: 23919745883ff9d60c18ce75fc1a056a017bce37
Parents: cd0015d
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 16:12:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 16:12:34 2015 -0600
----------------------------------------------------------------------
stack/corepersistence/queryindex/pom.xml | 8 +
.../usergrid/persistence/index/IndexFig.java | 7 +
.../index/IndexOperationMessage.java | 29 +++-
.../persistence/index/guice/IndexModule.java | 2 +
.../persistence/index/impl/BatchRequest.java | 41 +++++
.../index/impl/BufferQueueInMemory.java | 87 ----------
.../index/impl/BufferQueueInMemoryImpl.java | 87 ++++++++++
.../index/impl/BufferQueueSQSImpl.java | 158 +++++++++++++++++++
.../persistence/index/impl/DeIndexRequest.java | 106 +++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 39 ++---
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 37 ++---
.../persistence/index/impl/IndexRequest.java | 117 ++++++++++++++
.../index/guice/TestIndexModule.java | 6 +-
.../persistence/queue/QueueManager.java | 4 +-
15 files changed, 581 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index f6ae718..a5fbf6a 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -108,6 +108,14 @@
</dependency>
<dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
+
+ <dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index befbaa9..ce14449 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -55,6 +55,8 @@ public interface IndexFig extends GuicyFig {
public static final String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
+ public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
+
public static final String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
public static final String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
@@ -167,4 +169,9 @@ public interface IndexFig extends GuicyFig {
@Default("1000")
@Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
long getFailureRetryTime();
+
+ //give us 60 seconds to process the message
+ @Default("60")
+ @Key(INDEX_QUEUE_READ_TIMEOUT)
+ int getIndexQueueTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index 944a71f..43eaa01 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -17,23 +17,28 @@
package org.apache.usergrid.persistence.index;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.impl.BatchRequest;
+
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequestBuilder;
+import java.io.Serializable;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
* Container for index operations.
*/
-public class IndexOperationMessage {
- private final ConcurrentLinkedQueue<ActionRequestBuilder> builders;
+public class IndexOperationMessage implements Serializable {
+ private final Set<BatchRequest> builders;
private final BetterFuture<IndexOperationMessage> containerFuture;
public IndexOperationMessage(){
final IndexOperationMessage parent = this;
- builders = new ConcurrentLinkedQueue<>();
+ this.builders = new HashSet<>();
this.containerFuture = new BetterFuture<>(new Callable<IndexOperationMessage>() {
@Override
public IndexOperationMessage call() throws Exception {
@@ -42,7 +47,21 @@ public class IndexOperationMessage {
});
}
- public void addOperation(ActionRequestBuilder builder){
+
+ /**
+ * Add all our operations in the set
+ * @param requests
+ */
+ public void setOperations(final Set<BatchRequest> requests){
+ this.builders.addAll( requests);
+ }
+
+
+ /**
+ * Add the operation to the set
+ * @param builder
+ */
+ public void addOperation(BatchRequest builder){
builders.add(builder);
}
@@ -50,7 +69,7 @@ public class IndexOperationMessage {
* return operations for the message
* @return
*/
- public ConcurrentLinkedQueue<ActionRequestBuilder> getOperations(){
+ public Set<BatchRequest> getOperations(){
return builders;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index d911dab..b03e1c0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
import org.apache.usergrid.persistence.map.guice.MapModule;
+import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.safehaus.guicyfig.GuicyFigModule;
@@ -41,6 +42,7 @@ public abstract class IndexModule extends AbstractModule {
install(new GuicyFigModule(IndexFig.class));
install(new MapModule());
+ install(new QueueModule());
bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
new file mode 100644
index 0000000..df6c5b0
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.io.Serializable;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.client.Client;
+
+
+/**
+ * A batch request we can serialize and construct on receive
+ */
+public interface BatchRequest extends Serializable {
+
+
+ /**
+ * Passing the client and the bulk request, add ourselves to the bulk request
+ * @param client
+ * @param bulkRequest
+ */
+ public void doOperation(final Client client, final BulkRequestBuilder bulkRequest );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
deleted file mode 100644
index 403762f..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemory.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexOperationMessage;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class BufferQueueInMemory implements BufferQueue {
-
- private final ArrayBlockingQueue<IndexOperationMessage> messages;
-
-
- @Inject
- public BufferQueueInMemory(final IndexFig fig ) {
- messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
- }
-
-
- @Override
- public void offer( final IndexOperationMessage operation ) {
- messages.offer( operation );
- }
-
-
- @Override
- public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
-
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-
- //loop until we're we're full or we time out
- do {
- try {
- //we received 1, try to drain
- IndexOperationMessage polled = messages.poll( timeout, timeUnit );
-
- //drain
- if ( polled != null ) {
- response.add( polled );
- messages.drainTo( response, takeSize - response.size() );
- }
- }
- catch ( InterruptedException ie ) {
- //swallow
-
- }
- }
- while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
-
- return response;
- }
-
-
- @Override
- public void ack( final List<IndexOperationMessage> messages ) {
- //no op for this
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
new file mode 100644
index 0000000..ef0ef5f
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexOperationMessage;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class BufferQueueInMemoryImpl implements BufferQueue {
+
+ private final ArrayBlockingQueue<IndexOperationMessage> messages;
+
+
+ @Inject
+ public BufferQueueInMemoryImpl( final IndexFig fig ) {
+ messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+ messages.offer( operation );
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
+
+ final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+
+ //loop until we're we're full or we time out
+ do {
+ try {
+ //we received 1, try to drain
+ IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+
+ //drain
+ if ( polled != null ) {
+ response.add( polled );
+ messages.drainTo( response, takeSize - response.size() );
+ }
+ }
+ catch ( InterruptedException ie ) {
+ //swallow
+
+ }
+ }
+ while ( response.size() < takeSize && System.currentTimeMillis() < endTime );
+
+ return response;
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+ //no op for this
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
new file mode 100644
index 0000000..b814603
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexOperationMessage;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class BufferQueueSQSImpl implements BufferQueue {
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private final QueueManager queue;
+ private final IndexFig indexFig;
+
+
+ @Inject
+ public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig ) {
+ final QueueScope scope =
+ new QueueScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( scope );
+ this.indexFig = indexFig;
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+ final Message toQueue = new Message( operation.getOperations() );
+
+
+
+
+ try {
+ this.queue.sendMessage( toQueue );
+ operation.getFuture().run();
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //loop until we're we're full or we time out
+ List<QueueMessage> messages = queue
+ .getMessages( takeSize, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ Message.class );
+
+
+ final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
+
+ for ( final QueueMessage message : messages ) {
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message );
+
+ operation.setOperations( ( ( Message ) message.getBody() ).getData() );
+
+ response.add( operation );
+ }
+
+ return response;
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for(IndexOperationMessage ioe: messages){
+ toAck.add( ((SqsIndexOperationMessage)ioe).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ /**
+ * The message to queue to SQS
+ */
+ public static final class Message implements Serializable {
+ private final Set<BatchRequest> data;
+
+
+ private Message( final Set<BatchRequest> data ) {this.data = data;}
+
+
+ public Set<BatchRequest> getData() {
+ return data;
+ }
+ }
+
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexOperationMessage {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message ) {this.message = message;}
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
new file mode 100644
index 0000000..a279f16
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.Arrays;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.delete.DeleteRequestBuilder;
+import org.elasticsearch.client.Client;
+
+
+/**
+ * Represent the properties required to build an index request
+ */
+public class DeIndexRequest implements BatchRequest {
+
+ public final String[] indexes;
+ public final String entityType;
+ public final String documentId;
+
+
+ public DeIndexRequest( final String[] indexes, final String entityType, final String documentId) {
+ this.indexes = indexes;
+ this.entityType = entityType;
+ this.documentId = documentId;
+ }
+
+
+ @Override
+ public void doOperation(final Client client, final BulkRequestBuilder bulkRequest ){
+
+
+ for(final String index: indexes) {
+ final DeleteRequestBuilder builder = client.prepareDelete( index, entityType, documentId);
+
+ bulkRequest.add( builder );
+ }
+ }
+
+
+ public String[] getIndexes() {
+ return indexes;
+ }
+
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+
+ public String getDocumentId() {
+ return documentId;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final DeIndexRequest that = ( DeIndexRequest ) o;
+
+ if ( !documentId.equals( that.documentId ) ) {
+ return false;
+ }
+ if ( !entityType.equals( that.entityType ) ) {
+ return false;
+ }
+ if ( !Arrays.equals( indexes, that.indexes ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode( indexes );
+ result = 31 * result + entityType.hashCode();
+ result = 31 * result + documentId.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index d987b29..b0c731e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -73,10 +73,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final ApplicationScope applicationScope;
- private final Client client;
-
- private final boolean refresh;
-
private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
@@ -85,21 +81,17 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final AliasedEntityIndex entityIndex;
private IndexOperationMessage container;
- private final Timer batchTimer;
- public EsEntityIndexBatchImpl(final ApplicationScope applicationScope, final Client client,
+ public EsEntityIndexBatchImpl(final ApplicationScope applicationScope,
final IndexBufferProducer indexBatchBufferProducer,final IndexFig config,
- final AliasedEntityIndex entityIndex,final MetricsFactory metricsFactory ) {
+ final AliasedEntityIndex entityIndex ) {
this.applicationScope = applicationScope;
- this.client = client;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.entityIndex = entityIndex;
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
this.alias = indexIdentifier.getAlias();
- this.refresh = config.isForcedRefresh();
- this.batchTimer = metricsFactory.getTimer( EsEntityIndexBatchImpl.class, "entity.index.batch.timer" );
//constrained
this.container = new IndexOperationMessage();
}
@@ -133,9 +125,10 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
final String entityType = entity.getId().getType();
- IndexRequestBuilder builder =
- client.prepareIndex(alias.getWriteAlias(), entityType, indexId).setSource( entityAsMap );
- container.addOperation(builder);
+
+
+ container.addOperation(new IndexRequest(alias.getWriteAlias(), entityType, indexId, entityAsMap));
+
return this;
}
@@ -174,23 +167,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if(indexes == null ||indexes.length == 0){
indexes = new String[]{indexIdentifier.getIndex(null)};
}
- //get all indexes then flush everyone
- Timer.Context timeDeindex = batchTimer.time();
- Observable.from(indexes)
- .map(new Func1<String, Object>() {
- @Override
- public Object call(String index) {
- try {
- DeleteRequestBuilder builder = client.prepareDelete(index, entityType, indexId).setRefresh(refresh);
- container.addOperation(builder);
- }catch (Exception e){
- log.error("failed to deindex",e);
- throw e;
- }
- return index;
- }
- }).toBlocking().last();
- timeDeindex.stop();
+
+ container.addOperation( new DeIndexRequest( indexes, entityType, indexId ) );
+
log.debug("Deindexed Entity with index id " + indexId);
return this;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a1a8ca7..7bdb41a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -400,7 +400,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
- applicationScope, esProvider.getClient(),indexBatchBufferProducer, config, this, metricsFactory );
+ applicationScope, indexBatchBufferProducer, config, this );
return batch;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 45c12a1..2342398 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -26,7 +26,6 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexBufferConsumer;
-import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import org.elasticsearch.action.ActionRequestBuilder;
@@ -37,7 +36,6 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.support.replication.ShardReplicationOperationRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
import org.slf4j.Logger;
@@ -48,7 +46,6 @@ import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -88,6 +85,10 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
this.consumer = Observable.create( new Observable.OnSubscribe<IndexOperationMessage>() {
@Override
public void call( final Subscriber<? super IndexOperationMessage> subscriber ) {
+
+ //name our thread so it's easy to see
+ Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
+
List<IndexOperationMessage> drainList;
do {
try {
@@ -130,7 +131,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
while ( true );
}
- } ).subscribeOn( Schedulers.io() ).buffer( config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
+ } ).subscribeOn( Schedulers.newThread() ).buffer( config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
config.getIndexBufferSize() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
public void call( List<IndexOperationMessage> containerList ) {
@@ -157,37 +158,29 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
//process and flatten all the messages to builder requests
- Observable<ActionRequestBuilder> flattenMessages = Observable.from(operationMessages)
+ Observable<BatchRequest> flattenMessages = Observable.from(operationMessages)
.subscribeOn(Schedulers.io())
- .flatMap(new Func1<IndexOperationMessage, Observable<ActionRequestBuilder>>() {
+ .flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
@Override
- public Observable<ActionRequestBuilder> call(IndexOperationMessage operationMessage) {
- return Observable.from(operationMessage.getOperations());
+ public Observable<BatchRequest> call( IndexOperationMessage operationMessage ) {
+ return Observable.from( operationMessage.getOperations() );
}
- });
+ } );
//batch shard operations into a bulk request
flattenMessages
.buffer(config.getIndexBatchSize())
- .doOnNext(new Action1<List<ActionRequestBuilder>>() {
+ .doOnNext(new Action1<List<BatchRequest>>() {
@Override
- public void call(List<ActionRequestBuilder> builders) {
+ public void call(List<BatchRequest> builders) {
try {
final BulkRequestBuilder bulkRequest = initRequest();
- for (ActionRequestBuilder builder : builders) {
+ for (BatchRequest builder : builders) {
indexSizeCounter.dec();
- if (builder instanceof IndexRequestBuilder) {
- bulkRequest.add((IndexRequestBuilder) builder);
- }
- if (builder instanceof DeleteRequestBuilder) {
- bulkRequest.add((DeleteRequestBuilder) builder);
- }
- if(builder instanceof DeleteByQueryRequestBuilder){
- DeleteByQueryRequestBuilder deleteByQueryRequestBuilder = (DeleteByQueryRequestBuilder) builder;
- deleteByQueryRequestBuilder.get();
- }
+
+ builder.doOperation( client, bulkRequest );
}
sendRequest(bulkRequest);
}catch (Exception e){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
new file mode 100644
index 0000000..381d005
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.Map;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
+
+
+/**
+ * Represent the properties required to build an index request
+ */
+public class IndexRequest implements BatchRequest {
+
+ public final String writeAlias;
+ public final String entityType;
+ public final String documentId;
+
+ public final Map<String, Object> data;
+
+
+ public IndexRequest( final String writeAlias, final String entityType, final String documentId,
+ final Map<String, Object> data ) {
+ this.writeAlias = writeAlias;
+ this.entityType = entityType;
+ this.documentId = documentId;
+ this.data = data;
+ }
+
+
+ public void doOperation(final Client client, final BulkRequestBuilder bulkRequest ){
+ IndexRequestBuilder builder =
+ client.prepareIndex(writeAlias, entityType, documentId).setSource( data );
+
+
+ bulkRequest.add( builder );
+
+ }
+
+
+ public String getWriteAlias() {
+ return writeAlias;
+ }
+
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+
+ public String getDocumentId() {
+ return documentId;
+ }
+
+
+ public Map<String, Object> getData() {
+ return data;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final IndexRequest that = ( IndexRequest ) o;
+
+ if ( !data.equals( that.data ) ) {
+ return false;
+ }
+ if ( !documentId.equals( that.documentId ) ) {
+ return false;
+ }
+ if ( !entityType.equals( that.entityType ) ) {
+ return false;
+ }
+ if ( !writeAlias.equals( that.writeAlias ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = writeAlias.hashCode();
+ result = 31 * result + entityType.hashCode();
+ result = 31 * result + documentId.hashCode();
+ result = 31 * result + data.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7e2312d..57c2fab 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -23,7 +23,8 @@ import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueInMemory;
+import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
+import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
public class TestIndexModule extends TestModule {
@@ -37,7 +38,8 @@ public class TestIndexModule extends TestModule {
install( new IndexModule() {
@Override
public void wireBufferQueue() {
- bind( BufferQueue.class).to( BufferQueueInMemory.class );
+ bind( BufferQueue.class).to( BufferQueueInMemoryImpl.class );
+// bind( BufferQueue.class).to( BufferQueueSQSImpl.class );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23919745/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueManager.java
index 223860e..dd044d2 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueManager.java
@@ -29,8 +29,8 @@ public interface QueueManager {
/**
* Read messages from queue
* @param limit
- * @param transactionTimeout timeout in ms
- * @param waitTime wait time for next message in ms
+ * @param transactionTimeout timeout in seconds
+ * @param waitTime wait time for next message in milliseconds
* @param klass class to cast the return from
* @return List of Queue Messages
*/
[25/50] incubator-usergrid git commit: Changes the batch future set.
Posted by gr...@apache.org.
Changes the batch future set.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d81dfaa1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d81dfaa1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d81dfaa1
Branch: refs/heads/USERGRID-460
Commit: d81dfaa14f11c524e3f114b0b030bc447ee8100b
Parents: 77542c6
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:09:48 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:09:48 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d81dfaa1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 8481dab..16b3ff9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -197,9 +197,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* No-op, just disregard it
*/
if(tempContainer.isEmpty()){
- final BetterFuture<?> future = tempContainer.getFuture();
- future.done();
- return future;
+ tempContainer.done();
+ return tempContainer.getFuture();
}
return indexBatchBufferProducer.put(tempContainer);
[03/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c47f32a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c47f32a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c47f32a7
Branch: refs/heads/USERGRID-460
Commit: c47f32a72d3ab322985e66e893d1622af3be16fc
Parents: 2391974 31bee37
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 16:12:48 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 16:12:48 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c47f32a7/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
[33/50] incubator-usergrid git commit: Updated queuescope to be name
with no need for application id. We are only using them in our subsystems,
not applications.
Posted by gr...@apache.org.
Updated queuescope to be name with no need for application id. We are only using them in our subsystems, not applications.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/361060ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/361060ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/361060ec
Branch: refs/heads/USERGRID-460
Commit: 361060ec3c90488a8c944d2f30fc95310bd41453
Parents: 97634df
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 13 09:10:28 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 13 09:10:28 2015 -0600
----------------------------------------------------------------------
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 29 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
11 files changed, 231 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
new file mode 100644
index 0000000..ea3e046
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.guice;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
+import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
+import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class QueueProvider implements Provider<BufferQueue> {
+
+ private final IndexFig indexFig;
+
+ private final QueueManagerFactory queueManagerFactory;
+ private final MapManagerFactory mapManagerFactory;
+ private final MetricsFactory metricsFactory;
+
+ private BufferQueue bufferQueue;
+
+
+ @Inject
+ public QueueProvider( final IndexFig indexFig, final QueueManagerFactory queueManagerFactory,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ this.indexFig = indexFig;
+
+
+ this.queueManagerFactory = queueManagerFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.metricsFactory = metricsFactory;
+ }
+
+
+ @Override
+ @Singleton
+ public BufferQueue get() {
+ if ( bufferQueue == null ) {
+ bufferQueue = getQueue();
+ }
+
+
+ return bufferQueue;
+ }
+
+
+ private BufferQueue getQueue() {
+ final String value = indexFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new BufferQueueInMemoryImpl( indexFig );
+ case SQS:
+ return new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
index b2b2ec6..cf6bf24 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScope.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.queue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-public interface QueueScope extends ApplicationScope {
+public interface QueueScope {
/**
* Get the name of the the map
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
deleted file mode 100644
index 3a508a9..0000000
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueScopeFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.queue;
-
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import java.util.UUID;
-
-/**
- * Created by ApigeeCorporation on 10/23/14.
- */
-public interface QueueScopeFactory {
-
- public QueueScope getScope( UUID id, String queueName);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
index 4e487e1..1b2d6ea 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
@@ -17,15 +17,16 @@
*/
package org.apache.usergrid.persistence.queue.guice;
-import com.google.inject.AbstractModule;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+import org.safehaus.guicyfig.GuicyFigModule;
+
import org.apache.usergrid.persistence.queue.QueueFig;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
import org.apache.usergrid.persistence.queue.impl.SQSQueueManagerImpl;
-import org.safehaus.guicyfig.GuicyFigModule;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
/**
@@ -43,10 +44,8 @@ public class QueueModule extends AbstractModule {
install( new GuicyFigModule( QueueFig.class) );
// create a guice factory for getting our collection manager
- install(new FactoryModuleBuilder()
- .implement(QueueManager.class, SQSQueueManagerImpl.class)
- .build(QueueManagerFactory.class));
- bind( QueueScopeFactory.class ).to( QueueScopeFactoryImpl.class );
+ install( new FactoryModuleBuilder().implement( QueueManager.class, SQSQueueManagerImpl.class )
+ .build( QueueManagerFactory.class ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
deleted file mode 100644
index a145a6e..0000000
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeFactoryImpl.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.queue.impl;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.queue.QueueFig;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.QueueScopeFactory;
-
-import java.util.UUID;
-
-/**
- * Returns scope for queues.
- */
-public class QueueScopeFactoryImpl implements QueueScopeFactory {
-
- private QueueFig fig;
-
- @Inject
- public QueueScopeFactoryImpl(QueueFig fig){
- this.fig = fig;
- }
-
- //applicationId is always ManagementApplicationId so not really needed here.
- @Override
- public QueueScope getScope(UUID applicationId, String queueName) {
- return new QueueScopeImpl(new SimpleId(applicationId, fig.getPrefix()), queueName);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
index a928803..381cd8e 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueScopeImpl.java
@@ -24,18 +24,14 @@ import org.apache.usergrid.persistence.queue.QueueScope;
* Created by ApigeeCorporation on 10/3/14.
*/
public class QueueScopeImpl implements QueueScope {
- private final Id owner;
+
private final String name;
- public QueueScopeImpl( final Id owner, final String name ) {
- this.owner = owner;
+ public QueueScopeImpl( final String name ) {
this.name = name;
}
- @Override
- public Id getApplication() {
- return owner;
- }
+
@Override
@@ -57,9 +53,7 @@ public class QueueScopeImpl implements QueueScope {
if ( !name.equals( queueScope.name ) ) {
return false;
}
- if ( !owner.equals( queueScope.owner ) ) {
- return false;
- }
+
return true;
}
@@ -67,17 +61,6 @@ public class QueueScopeImpl implements QueueScope {
@Override
public int hashCode() {
- int result = owner.hashCode();
- result = 31 * result + name.hashCode();
- return result;
- }
-
-
- @Override
- public String toString() {
- return "QueueScopeImpl{" +
- "owner=" + owner +
- ", name='" + name + '\'' +
- '}';
+ return name.hashCode();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index 1fbd9b6..088359a 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -54,6 +54,7 @@ import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -124,7 +125,10 @@ public class SQSQueueManagerImpl implements QueueManager {
private String getName() {
- String name = fig.getPrefix() + "_" + scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
+ String name = fig.getPrefix() + "_" + scope.getName();
+
+ Preconditions.checkArgument(name.length() <= 80, "Your name must be < than 80 characters");
+
return name;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
new file mode 100644
index 0000000..45218c2
--- /dev/null
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.queue;
+
+
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.amazonaws.AmazonClientException;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule implements TestRule {
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+ try {
+ base.evaluate();
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ if ( t instanceof AmazonClientException ) {
+
+ final AmazonClientException ace = ( AmazonClientException ) t;
+
+ if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+ "could not get aws secret key from system properties" ) ) {
+ //swallow
+ return true;
+ }
+ }
+
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index 935fd16..eecb9e1 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -25,24 +25,24 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
@RunWith( ITRunner.class )
@UseModules( { TestQueueModule.class } )
@@ -53,8 +53,12 @@ public class QueueManagerTest {
@Inject
protected QueueManagerFactory qmf;
- @Inject
- protected QueueScopeFactory queueScopeFactory;
+ /**
+ * Mark tests as ignored if now AWS creds are present
+ */
+ @Rule
+ public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+
protected QueueScope scope;
private QueueManager qm;
@@ -62,21 +66,11 @@ public class QueueManagerTest {
@Before
public void mockApp() {
- this.scope = new QueueScopeImpl( new SimpleId( "application" ), "testQueue" );
+ this.scope = new QueueScopeImpl( "testQueue" );
qm = qmf.getQueueManager(scope);
- queueScopeFactory = new QueueScopeFactoryImpl(queueFig);
}
- @Test
- public void scopeFactory(){
- UUID uuid = UUID.randomUUID();
- String key = "test";
- QueueScope scope =queueScopeFactory.getScope(uuid,key);
- assertEquals(key,scope.getName());
- assertEquals(scope.getApplication().getUuid(),uuid);
- }
- @Ignore("need aws creds")
@Test
public void send() throws IOException,ClassNotFoundException{
String value = "bodytest";
@@ -92,7 +86,6 @@ public class QueueManagerTest {
}
- @Ignore("need aws creds")
@Test
public void sendMore() throws IOException,ClassNotFoundException{
HashMap<String,String> values = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index d34cc22..e45b82a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
import org.apache.usergrid.services.ServiceManager;
import org.apache.usergrid.services.ServiceManagerFactory;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
@@ -47,7 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
public final int MESSAGE_TRANSACTION_TIMEOUT = 25 * 1000;
private final QueueManagerFactory queueManagerFactory;
- private final QueueScopeFactory queueScopeFactory;
public long DEFAULT_SLEEP = 5000;
@@ -84,7 +84,6 @@ public class QueueListener {
this.emf = emf;
this.metricsService = smf.getApplicationContext().getBean( Injector.class ).getInstance(MetricsFactory.class);
this.properties = props;
- this.queueScopeFactory = smf.getApplicationContext().getBean( Injector.class ).getInstance(QueueScopeFactory.class);
}
/**
@@ -146,7 +145,7 @@ public class QueueListener {
com.codahale.metrics.Timer timer = metricsService.getTimer(QueueListener.class, "dequeue");
svcMgr = smf.getServiceManager(smf.getManagementAppId());
LOG.info("getting from queue {} ", queueName);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), queueName);
+ QueueScope queueScope = new QueueScopeImpl( queueName ) {};
QueueManager queueManager = TEST_QUEUE_MANAGER != null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
// run until there are no more active jobs
long runCount = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/361060ec/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
index 8277e86..b87904e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
import org.apache.usergrid.services.ServiceManager;
import org.apache.usergrid.services.ServiceManagerFactory;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
@@ -48,7 +49,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class QueueListener {
public final int MESSAGE_TRANSACTION_TIMEOUT = 25 * 1000;
private final QueueManagerFactory queueManagerFactory;
- private final QueueScopeFactory queueScopeFactory;
public long DEFAULT_SLEEP = 5000;
@@ -93,7 +93,6 @@ public abstract class QueueListener {
this.emf = injector.getInstance( EntityManagerFactory.class ); //emf;
this.metricsService = injector.getInstance(MetricsFactory.class);
this.properties = props;
- this.queueScopeFactory = injector.getInstance(QueueScopeFactory.class);
}
@@ -165,7 +164,7 @@ public abstract class QueueListener {
com.codahale.metrics.Timer timer = metricsService.getTimer(QueueListener.class, "dequeue");
svcMgr = smf.getServiceManager(smf.getManagementAppId());
LOG.info("getting from queue {} ", queueName);
- QueueScope queueScope = queueScopeFactory.getScope(smf.getManagementAppId(), queueName);
+ QueueScope queueScope = new QueueScopeImpl( queueName);
QueueManager queueManager = TEST_QUEUE_MANAGER != null ? TEST_QUEUE_MANAGER : queueManagerFactory.getQueueManager(queueScope);
// run until there are no more active jobs
long runCount = 0;
[49/50] incubator-usergrid git commit: Merge branch 'USERGRID-490'
into two-dot-o
Posted by gr...@apache.org.
Merge branch 'USERGRID-490' into two-dot-o
* USERGRID-490:
Added more descriptive error message
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/eb0c689d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/eb0c689d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/eb0c689d
Branch: refs/heads/USERGRID-460
Commit: eb0c689da0587bf1c04235f97c0c03ea3145152a
Parents: aa044cf 3a47cf3
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 18 10:07:33 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 18 10:07:33 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[08/50] incubator-usergrid git commit: add timer and meter to update
entity
Posted by gr...@apache.org.
add timer and meter to update entity
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/06fe6097
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/06fe6097
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/06fe6097
Branch: refs/heads/USERGRID-460
Commit: 06fe6097cb854fa35e9071c87efe8cb7ec8f9f88
Parents: 56c5701
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 08:49:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 08:49:13 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/06fe6097/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1663756..26eff4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -33,6 +33,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
+import com.codahale.metrics.Meter;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,6 +207,8 @@ public class CpEntityManager implements EntityManager {
private Timer entGetEntityCountersTimer;
private Timer esIndexEntityCollectionTimer;
private Timer entRevokeRolePermissionsTimer;
+ private Timer updateEntityTimer;
+ private Meter updateEntityMeter;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -267,6 +270,9 @@ public class CpEntityManager implements EntityManager {
this.entRevokeRolePermissionsTimer =
this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+ this.updateEntityMeter =this.metricsFactory.getMeter(CpEntityManager.class,"cp.entity.update.meter");
+ this.updateEntityTimer =this.metricsFactory.getTimer(CpEntityManager.class, "cp.entity.update.timer");
+
// set to false for now
this.skipAggregateCounters = false;
@@ -562,6 +568,9 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull(appId,"app scope should never be null");
// first, update entity index in its own collection scope
+ updateEntityMeter.mark();
+ Timer.Context timer = updateEntityTimer.time();
+
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(appId, type );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -618,6 +627,7 @@ public class CpEntityManager implements EntityManager {
// update in all containing collections and connection indexes
CpRelationManager rm = ( CpRelationManager ) getRelationManager( entity );
rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
+ timer.stop();
}
[38/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Conflicts:
stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9d00c223
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9d00c223
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9d00c223
Branch: refs/heads/USERGRID-460
Commit: 9d00c2235111954abbbcf9e26afacca74df1c1bd
Parents: a1b557d 361060e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 12:23:53 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 12:23:53 2015 -0600
----------------------------------------------------------------------
.../main/groovy/configure_elasticsearch.groovy | 1 -
.../persistence/index/guice/QueueProvider.java | 116 +++++++++++++++++++
.../usergrid/persistence/queue/QueueScope.java | 2 +-
.../persistence/queue/QueueScopeFactory.java | 34 ------
.../persistence/queue/guice/QueueModule.java | 17 ++-
.../queue/impl/QueueScopeFactoryImpl.java | 48 --------
.../persistence/queue/impl/QueueScopeImpl.java | 27 +----
.../queue/impl/SQSQueueManagerImpl.java | 6 +-
.../persistence/queue/NoAWSCredsRule.java | 81 +++++++++++++
.../persistence/queue/QueueManagerTest.java | 31 ++---
.../services/notifications/QueueListener.java | 5 +-
.../usergrid/services/queues/QueueListener.java | 5 +-
12 files changed, 232 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
[34/50] incubator-usergrid git commit: add replay strategy
Posted by gr...@apache.org.
add replay strategy
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a1b557d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a1b557d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a1b557d0
Branch: refs/heads/USERGRID-460
Commit: a1b557d073fd8dbe4c20f949ce7497db01d7cea5
Parents: fde59b7
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 13 11:25:52 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 13 11:25:52 2015 -0600
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 167 ++++++++++++-------
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 6 +-
.../persistence/core/astyanax/CassandraFig.java | 4 +-
3 files changed, 111 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 1c30e75..5f96b92 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -25,7 +25,14 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.hystrix.Hystrix;
+import com.netflix.hystrix.HystrixCommand;
+import com.netflix.hystrix.HystrixCommandGroupKey;
+import com.netflix.hystrix.HystrixThreadPoolProperties;
import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,12 +73,14 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
protected final SerializationFig serializationFig;
protected final Keyspace keyspace;
+ private final CassandraConfig cassandraFig;
@Inject
public WriteUniqueVerify( final UniqueValueSerializationStrategy uniqueValueSerializiationStrategy,
- final SerializationFig serializationFig, final Keyspace keyspace ) {
+ final SerializationFig serializationFig, final Keyspace keyspace, final CassandraConfig cassandraFig ) {
this.keyspace = keyspace;
+ this.cassandraFig = cassandraFig;
Preconditions.checkNotNull( uniqueValueSerializiationStrategy, "uniqueValueSerializationStrategy is required" );
Preconditions.checkNotNull( serializationFig, "serializationFig is required" );
@@ -88,100 +97,134 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final MvccEntity mvccEntity = ioevent.getEvent();
- final Id entityId = mvccEntity.getId();
-
- final UUID entityVersion = mvccEntity.getVersion();
-
-
final Entity entity = mvccEntity.getEntity().get();
-
final CollectionScope scope = ioevent.getEntityCollection();
// use simple thread pool to verify fields in parallel
+ ConsistentReplayCommand cmd = new ConsistentReplayCommand(uniqueValueStrat,keyspace,serializationFig,cassandraFig,scope,entity);
+ Map<String,Field> uniquenessViolations = cmd.execute();
+ //We have violations, throw an exception
+ if ( !uniquenessViolations.isEmpty() ) {
+ throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
+ }
+ }
- final Collection<Field> entityFields = entity.getFields();
+ private static class ConsistentReplayCommand extends HystrixCommand<Map<String,Field>>{
+
+ private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ private final Keyspace keySpace;
+ private final SerializationFig serializationFig;
+ private final CassandraConfig fig;
+ private final CollectionScope scope;
+ private final Entity entity;
+
+ public ConsistentReplayCommand(UniqueValueSerializationStrategy uniqueValueSerializationStrategy,Keyspace keySpace, SerializationFig serializationFig, CassandraConfig fig,CollectionScope scope, Entity entity){
+ super(REPLAY_GROUP);
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.keySpace = keySpace;
+ this.serializationFig = serializationFig;
+ this.fig = fig;
+ this.scope = scope;
+ this.entity = entity;
+ }
- //allocate our max size, worst case
- final List<Field> uniqueFields = new ArrayList<>( entityFields.size() );
+ @Override
+ protected Map<String, Field> run() throws Exception {
+ return executeStrategy(fig.getReadCL());
+ }
- final MutationBatch batch = keyspace.prepareMutationBatch();
- //
- // Construct all the functions for verifying we're unique
- //
- for ( final Field field : entityFields ) {
+ @Override
+ protected Map<String, Field> getFallback() {
+ return executeStrategy(fig.getConsistentReadCL());
+ }
- // if it's unique, create a function to validate it and add it to the list of
- // concurrent validations
- if ( field.isUnique() ) {
+ public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
+ Collection<Field> entityFields = entity.getFields();
+ //allocate our max size, worst case
+ final List<Field> uniqueFields = new ArrayList<Field>( entityFields.size() );
+ //now get the set of fields back
+ final UniqueValueSet uniqueValues;
+ //todo add consistencylevel and read back if fail using
+ try {
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, entityId, entityVersion );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to read from cassandra", e );
+ }
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
- batch.mergeShallow( mb );
+ final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
- uniqueFields.add( field );
- }
- }
+ //loop through each field that was unique
+ for ( final Field field : uniqueFields ) {
- //short circuit nothing to do
- if ( uniqueFields.size() == 0 ) {
- return;
- }
+ final UniqueValue uniqueValue = uniqueValues.getValue( field.getName() );
+ if ( uniqueValue == null ) {
+ throw new RuntimeException(
+ String.format( "Could not retrieve unique value for field %s, unable to verify",
+ field.getName() ) );
+ }
- //perform the write
- try {
- batch.execute();
- }
- catch ( ConnectionException ex ) {
- throw new RuntimeException( "Unable to write to cassandra", ex );
- }
+ final Id returnedEntityId = uniqueValue.getEntityId();
- //now get the set of fields back
- final UniqueValueSet uniqueValues;
- try {
- uniqueValues = uniqueValueStrat.load( scope, uniqueFields );
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to read from cassandra", e );
- }
+ if ( !entity.getId().equals(returnedEntityId) ) {
+ uniquenessViolations.put( field.getName(), field );
+ }
+ }
+ final MutationBatch batch = keySpace.prepareMutationBatch();
+ //
+ // Construct all the functions for verifying we're unique
+ //
+ for ( final Field field : entity.getFields() ) {
+ // if it's unique, create a function to validate it and add it to the list of
+ // concurrent validations
+ if ( field.isUnique() ) {
- final Map<String, Field> uniquenessViolations = new HashMap<>( uniqueFields.size() );
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, entity.getId(), entity.getVersion() );
- //loop through each field that was unique
- for ( final Field field : uniqueFields ) {
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueSerializationStrategy.write( scope, written, serializationFig.getTimeout() );
- final UniqueValue uniqueValue = uniqueValues.getValue( field.getName() );
+ batch.mergeShallow( mb );
- if ( uniqueValue == null ) {
- throw new RuntimeException(
- String.format( "Could not retrieve unique value for field %s, unable to verify",
- field.getName() ) );
- }
+ uniqueFields.add(field);
+ }
+ }
- final Id returnedEntityId = uniqueValue.getEntityId();
+ //short circuit nothing to do
+ if ( uniqueFields.size() == 0 ) {
+ return uniquenessViolations ;
+ }
- if ( !entityId.equals( returnedEntityId ) ) {
- uniquenessViolations.put( field.getName(), field );
+ //perform the write
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException ex ) {
+ throw new RuntimeException( "Unable to write to cassandra", ex );
}
- }
-
- //We have violations, throw an exception
- if ( !uniquenessViolations.isEmpty() ) {
- throw new WriteUniqueVerifyException( mvccEntity, ioevent.getEntityCollection(), uniquenessViolations );
+ return uniquenessViolations;
}
}
+
+ /**
+ * Command group used for realtime user commands
+ */
+ public static final HystrixCommand.Setter
+ REPLAY_GROUP = HystrixCommand.Setter.withGroupKey(
+ HystrixCommandGroupKey.Factory.asKey( "user" ) ).andThreadPoolPropertiesDefaults(
+ HystrixThreadPoolProperties.Setter().withCoreSize( 1000 ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 4ebfd24..51cb198 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.persistence.collection.mvcc.stage.write;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Rule;
import org.junit.Test;
@@ -64,7 +65,8 @@ public class WriteUniqueVerifyTest {
@Inject
private SerializationFig fig;
-
+ @Inject
+ private CassandraConfig cassandraConfig;
@Test
@@ -81,7 +83,7 @@ public class WriteUniqueVerifyTest {
final MvccEntity mvccEntity = fromEntity( entity );
// run the stage
- WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace );
+ WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace,cassandraConfig );
newStage.call(
new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b557d0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index 66a716c..a907702 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -79,11 +79,11 @@ public interface CassandraFig extends GuicyFig {
@Default( "false" )
boolean isEmbedded();
- @Default("CL_ONE")
+ @Default("CL_LOCAL_ONE")
@Key(READ_CL)
String getReadCL();
- @Default("CL_LOCAL_ONE")
+ @Default("CL_LOCAL_QUORUM")
@Key(READ_CONSISTENT_CL)
String getConsistentReadCL();
[10/50] incubator-usergrid git commit: Added a master only elastic
search node
Posted by gr...@apache.org.
Added a master only elastic search node
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e58597be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e58597be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e58597be
Branch: refs/heads/USERGRID-460
Commit: e58597be99d18ed1f3bfa4387746bdc0254ff122
Parents: b3e42dd
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 10:14:09 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 10:14:09 2015 -0600
----------------------------------------------------------------------
.../dist/init_instance/install_elasticsearch.sh | 12 +-
.../main/groovy/configure_elasticsearch.groovy | 70 ++--
stack/awscluster/ugcluster-cf.json | 326 ++++++++++++++++++-
.../events/EntityDeletedHandler.java | 2 +-
.../events/EntityVersionCreatedHandler.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 26 +-
.../index/impl/EsEntityIndexImpl.java | 166 +++++-----
.../index/impl/EsIndexBufferConsumerImpl.java | 5 +-
8 files changed, 476 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index 06af849..b2018e1 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -42,9 +42,17 @@ echo "vm.swappiness = 0" >> /etc/sysctl.conf
sysctl -p
# No need to do this, elasticsearch nodes are also cassandra nodes
+
cd /usr/share/usergrid/scripts
-groovy registry_register.groovy elasticsearch
-groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
+
+#If we're the master, register ourselves and move on, if we're not, wait for the master to come up
+if [ "$ES_MASTER" = "true" ]; then
+ groovy registry_register.groovy elasticsearch_master
+else
+ groovy registry_register.groovy elasticsearch
+ groovy wait_for_instances.groovy elasticsearch_master 1
+fi
+
# leave room for Cassandra: use about one half of RAM for heap
case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 7bee82c..19c969d 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -18,10 +18,10 @@
-//
-// configure_elasticsearch.groovy
-//
-// Emits Elasticsearch config file based on environment and Elasticsearch node
+//
+// configure_elasticsearch.groovy
+//
+// Emits Elasticsearch config file based on environment and Elasticsearch node
// registry in SimpleDB
//
@@ -29,13 +29,18 @@
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def clusterName = (String)System.getenv().get("ES_CLUSTER_NAME")
+def isMaster = ((String)System.getenv().get("ES_MASTER")).toBoolean()
+
int esNumServers = ((String)System.getenv().get("ES_NUM_SERVERS")).toInteger()
-int quorum = esNumServers/2+1;
+///int quorum = esNumServers/2+1;
+
+//TODO get this from the number of master nodes
+int quorum = 1
NodeRegistry registry = new NodeRegistry();
// build seed list by listing all Elasticsearch nodes found in SimpleDB domain with our stackName
-def selectResult = registry.searchNode('elasticsearch')
+def selectResult = registry.searchNode('elasticsearch_master')
def esnodes = ""
def sep = ""
for (hostname in selectResult) {
@@ -43,13 +48,19 @@ for (hostname in selectResult) {
sep = ","
}
+
+def nodeData = !isMaster
+def nodeMaster = isMaster
+
+
+
def elasticSearchConfig = """
cluster.name: ${clusterName}
discovery.zen.minimum_master_nodes: ${quorum}
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: [${esnodes}]
node:
- name: ${hostName}
+ name: ${hostName}
network:
host: ${hostName}
path:
@@ -65,21 +76,20 @@ es.logger.level: INFO
# Temporarily removing. We don't know better :)
# http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_don_8217_t_touch_these_settings.html#_threadpools
#
-#threadpool:
-# index:
-# type: fixed
-# size: 160
-# queue_size: 0
-# bulk:
-# type: fixed
-# size: 5000
-# size: 16
-# queue_size: 100
-# search:
-# size: 10000
-# size: 48
-# type: fixed
-# queue_size: 100
+threadpool:
+ index:
+ type: fixed
+ size: 160
+ queue_size: 1000
+ bulk:
+ type: fixed
+ size: 160
+ size: 16
+ queue_size: 1000
+ search:
+ size: 320
+ type: fixed
+ queue_size: 1000
action.auto_create_index: false
@@ -108,10 +118,18 @@ cluster.routing.allocation.disk.watermark.low: .97
cluster.routing.allocation.disk.watermark.high: .99
#Set streaming high water marks so reboots don't kill our service
-cluster.routing.allocation.node_concurrent_recoveries: 4
-cluster.routing.allocation.node_initial_primaries_recoveries: 18
-indices.recovery.concurrent_streams: 4
-indices.recovery.max_bytes_per_sec: 40mb
+cluster.routing.allocation.node_concurrent_recoveries: 40
+cluster.routing.allocation.node_initial_primaries_recoveries: 40
+indices.recovery.concurrent_streams: 16
+indices.recovery.max_bytes_per_sec: 300mb
+
+
+##############################
+# Master or data node options
+#############################
+
+node.data: ${nodeData}
+node.master: ${nodeMaster}
###############
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 7926929..0c2a9d2 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -109,6 +109,14 @@
"Default": "6",
"MinValue": "2"
},
+ "ESNumMasterServers": {
+ "Description": "Number of ES master servers to start.",
+ "Type": "Number",
+ "Default": "1",
+ "MinValue": "1"
+ },
+
+
"ESInstanceType": {
"Description": "Instance type for ES servers",
"Type": "String",
@@ -947,7 +955,7 @@
}
}
},
- "ESAutoScalingLaunchConfiguration":{
+ "ESMasterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -979,7 +987,10 @@
"\n",
"export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
"export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "export ES_MASTER=true", "\n",
+
"export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+
"\n",
"export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
"\n",
@@ -1062,7 +1073,7 @@
]
}
},
- "ESAutoScalingGroup": {
+ "ESMasterAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
"Properties": {
@@ -1105,13 +1116,13 @@
}
],
"LaunchConfigurationName": {
- "Ref": "ESAutoScalingLaunchConfiguration"
+ "Ref": "ESMasterAutoScalingLaunchConfiguration"
},
"MinSize": {
- "Ref": "ESNumServers"
+ "Ref": "ESNumMasterServers"
},
"MaxSize": {
- "Ref": "ESNumServers"
+ "Ref": "ESNumMasterServers"
},
"NotificationConfiguration": {
"TopicARN": {
@@ -1251,6 +1262,311 @@
]
}
},
+ "ESAutoScalingLaunchConfiguration":{
+ "Type":"AWS::AutoScaling::LaunchConfiguration",
+ "Properties":{
+ "UserData":{
+ "Fn::Base64":{
+ "Fn::Join":[
+ "",
+ [
+ "#!/bin/bash -ex\n",
+ "# ES NODE STARTUP \n",
+ "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
+ "\n",
+ "mkdir -p /usr/share/usergrid\n",
+ "\n",
+ "# create script that sets our environment variables\n",
+ "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
+ "alias sudo='sudo -E'\n", "\n",
+ "export TYPE=es\n",
+ "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n", "\n",
+ "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
+ "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
+ "\n",
+ "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
+ "export EC2_REGION=", { "Ref":"AWS::Region" },
+ "\n",
+ "export EC2_URL=https://ec2.amazonaws.com/\n",
+ "\n",
+ "export ES_SECURITY_GROUP_NAME=", { "Ref":"ESSecurityGroup" }, "\n",
+ "\n",
+ "\n",
+ "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
+ "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "export ES_MASTER=false","\n",
+ "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+ "\n",
+ "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
+ "\n",
+ "EOF\n",
+ "\n",
+ "# put AWS creds in environment\n",
+ "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
+ "export AWS_ACCESS_KEY=", { "Ref":"ESKey" }, "\n",
+ "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "\n",
+ "# setup s3cmd (will be installed by init script) \n",
+ "cat >/etc/s3cfg <<EOF\n",
+ "access_key=", { "Ref":"ESKey" }, "\n",
+ "secret_key=", { "Fn::GetAtt":[ "ESKey", "SecretAccessKey" ] },
+ "\n",
+ "EOF\n",
+ "chmod 644 /etc/s3cfg\n",
+ "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
+ "ln -s /etc/s3cfg ~root/.s3cfg\n",
+ "\n",
+ "# download usergrid and init script bundle from S3\n",
+ "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
+ "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
+ "apt-get update\n",
+ "apt-get -y install s3cmd\n",
+ "cd /usr/share/usergrid\n",
+ "s3cmd --config=/etc/s3cfg get s3://", { "Ref":"ReleaseBucket" }, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "chmod 755 ./init_instance/*.sh\n",
+ "cd ./init_instance\n",
+ "# init as an ES node \n",
+ "sh ./init_es_server.sh\n"
+ ]
+ ]
+ }
+ },
+ "KeyName": {
+ "Ref": "KeyPair"
+ },
+ "ImageId": {
+ "Fn::FindInMap": [
+ "AWSRegionArch2AMI",
+ {
+ "Ref": "AWS::Region"
+ },
+ {
+ "Fn::FindInMap": [
+ "AWSInstanceType2Arch",
+ {
+ "Ref": "CassInstanceType"
+ },
+ "Arch"
+ ]
+ }
+ ]
+ },
+ "InstanceType": {
+ "Ref": "ESInstanceType"
+ },
+ "IamInstanceProfile": {
+ "Ref": "RootInstanceProfile"
+ },
+ "SecurityGroups": [
+ {
+ "Ref": "ESSecurityGroup"
+ }
+ ],
+
+ "BlockDeviceMappings": [
+ {
+ "DeviceName": "/dev/sdb",
+ "VirtualName": "ephemeral0"
+ },
+ {
+ "DeviceName": "/dev/sdc",
+ "VirtualName": "ephemeral1"
+ }
+ ]
+ }
+ },
+ "ESAutoScalingGroup": {
+ "Type": "AWS::AutoScaling::AutoScalingGroup",
+ "Version": "2009-05-15",
+ "Properties": {
+ "AvailabilityZones": [
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ1"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ2"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ3"
+ ]
+ },
+ {
+ "Fn::FindInMap": [
+ "FourAZs",
+ {
+ "Ref": "AWS::Region"
+ },
+ "AZ4"
+ ]
+ }
+ ],
+ "LaunchConfigurationName": {
+ "Ref": "ESAutoScalingLaunchConfiguration"
+ },
+ "MinSize": {
+ "Ref": "ESNumServers"
+ },
+ "MaxSize": {
+ "Ref": "ESNumServers"
+ },
+ "NotificationConfiguration": {
+ "TopicARN": {
+ "Ref": "NotificationTopic"
+ },
+ "NotificationTypes": [
+ "autoscaling:EC2_INSTANCE_LAUNCH",
+ "autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
+ "autoscaling:EC2_INSTANCE_TERMINATE",
+ "autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
+ ]
+ }
+ }
+ },
+ "RestAutoScalingLaunchConfiguration":{
+ "Type":"AWS::AutoScaling::LaunchConfiguration",
+ "Properties":{
+ "UserData":{
+ "Fn::Base64":{
+ "Fn::Join":[
+ "",
+ [
+ "#!/bin/bash -ex\n",
+ "# REST SERVER STARTUP \n",
+ "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
+ "\n",
+ "mkdir -p /usr/share/usergrid\n",
+ "\n",
+ "# create script that sets our environment variables\n",
+ "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
+ "alias sudo='sudo -E'\n",
+ "\n",
+ "export TYPE=rest\n",
+ "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n",
+ "export YOURKIT=", { "Ref":"InstallYourkit" }, "\n",
+ "export DNS_NAME=", { "Ref":"DnsSubDomain" }, "\n",
+ "export DNS_DOMAIN=", { "Ref":"DnsDomain" }, "\n",
+ "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
+ "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
+ "export ELB_NAME=", { "Ref":"RestElasticLoadBalancer" }, "\n",
+ "\n",
+ "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
+ "export EC2_REGION=", { "Ref":"AWS::Region" }, "\n",
+ "export EC2_URL=https://ec2.amazonaws.com/\n", "\n",
+ "export REST_SECURITY_GROUP_NAME=", { "Ref":"RestSecurityGroup" }, "\n",
+ "export DB_SECURITY_GROUP_NAME=", { "Ref":"CassSecurityGroup" }, "\n",
+ "\n",
+ "export CASSANDRA_CLUSTER_NAME=", { "Ref":"CassClusterName" }, "\n",
+ "export CASSANDRA_KEYSPACE_NAME=usergrid", "\n",
+ "export CASSANDRA_NUM_SERVERS=", { "Ref":"CassNumServers" }, "\n",
+ "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
+ "export TOMCAT_NUM_SERVERS=", { "Ref":"RestMinServers" }, "\n",
+ "\n",
+ "export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor" }, "\n",
+ "\n",
+ "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency" }, "\n",
+ "\n",
+ "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
+ "\n",
+
+ "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
+ "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
+ "\n",
+ "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
+ "\n",
+ "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
+ "\n",
+ "export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
+ "export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
+ "\n",
+ "EOF\n",
+ "\n",
+ "# put AWS creds in environment\n",
+ "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
+ "export AWS_ACCESS_KEY=", { "Ref":"RestKey" }, "\n",
+ "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "\n",
+ "# setup s3cmd (will be installed by init script) \n",
+ "cat >/etc/s3cfg <<EOF\n",
+ "access_key=", { "Ref":"RestKey" }, "\n",
+ "secret_key=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
+ "EOF\n",
+ "chmod 644 /etc/s3cfg\n",
+ "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
+ "ln -s /etc/s3cfg ~root/.s3cfg\n",
+ "\n",
+ "# download usergrid and init script bundle from S3\n",
+ "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
+ "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
+ "apt-get update\n",
+ "apt-get -y install s3cmd\n",
+ "cd /usr/share/usergrid\n",
+ "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/ROOT.war\n",
+ "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
+ "mv ROOT.war webapps/ROOT.war\n",
+ "chmod 755 ./init_instance/*.sh\n",
+ "cd ./init_instance\n",
+ "# Init as a REST intance \n",
+ "sh ./init_rest_server.sh\n"
+ ]
+ ]
+ }
+ },
+ "KeyName":{
+ "Ref":"KeyPair"
+ },
+ "ImageId":{
+ "Fn::FindInMap":[
+ "AWSRegionArch2AMI",
+ {
+ "Ref":"AWS::Region"
+ },
+ {
+ "Fn::FindInMap":[
+ "AWSInstanceType2Arch",
+ {
+ "Ref":"RestInstanceType"
+ },
+ "Arch"
+ ]
+ }
+ ]
+ },
+ "InstanceType":{
+ "Ref":"RestInstanceType"
+ },
+ "IamInstanceProfile":{
+ "Ref":"RootInstanceProfile"
+ },
+ "SecurityGroups":[
+ {
+ "Ref":"RestSecurityGroup"
+ }
+ ]
+ }
+ },
"RestAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index ad4ed32..33cc988 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -63,6 +63,6 @@ public class EntityDeletedHandler implements EntityDeleted {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- ei.deleteAllVersionsOfEntity( entityId );
+// ei.deleteAllVersionsOfEntity( entityId );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 3b7163c..590df61 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -64,6 +64,6 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
+// ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 3edde4d..aa2bf8e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -74,19 +74,19 @@ public interface EntityIndex {
*/
public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id);
- /**
- * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
- * effectively removing all versions of an entity from all index scopes
- * @param entityId The entityId to remove
- */
- public Future deleteAllVersionsOfEntity(final Id entityId );
-
- /**
- * Takes all the previous versions of the current entity and deletes all previous versions
- * @param id The id to remove
- * @param version The max version to retain
- */
- public Future deletePreviousVersions(final Id id, final UUID version);
+// /**
+// * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
+// * effectively removing all versions of an entity from all index scopes
+// * @param entityId The entityId to remove
+// */
+// public Future deleteAllVersionsOfEntity(final Id entityId );
+//
+// /**
+// * Takes all the previous versions of the current entity and deletes all previous versions
+// * @param id The id to remove
+// * @param version The max version to retain
+// */
+// public Future deletePreviousVersions(final Id id, final UUID version);
/**
* Refresh the index.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 99d2b84..99643da 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -662,89 +662,89 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
- @Override
- public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
-
- String idString = IndexingUtils.idString(entityId).toLowerCase();
-
- final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
-
- //Added For Graphite Metrics
- final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
- final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
- final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
- .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
-
- response.addListener( new ActionListener<DeleteByQueryResponse>() {
-
- @Override
- public void onResponse( DeleteByQueryResponse response) {
- timeDeleteAllVersions.stop();
- logger
- .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
- entityId.getUuid(), response.status().toString() );
-
- checkDeleteByQueryResponse(tqb, response);
- }
-
-
- @Override
- public void onFailure( Throwable e ) {
- timeDeleteAllVersions.stop();
- logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
- entityId.getUuid(), e);
-
-
- }
- });
- timeDeleteAllVersionsFuture.stop();
- return response;
- }
-
-
- @Override
- public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
-
- String idString = IndexingUtils.idString( entityId ).toLowerCase();
-
- final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
- FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
- );
-
- //Added For Graphite Metrics
- //Checks the time from the execute to the response below
- final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
- final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
- final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
- .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
-
- //Added For Graphite Metrics
- response.addListener(new ActionListener<DeleteByQueryResponse>() {
- @Override
- public void onResponse(DeleteByQueryResponse response) {
- timeDeletePreviousVersions.stop();
- //error message needs to be retooled so that it describes the entity more throughly
- logger
- .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
- entityId.getType(), entityId.getUuid(), version, response.status().toString());
-
- checkDeleteByQueryResponse( fqb, response );
- }
-
-
- @Override
- public void onFailure( Throwable e ) {
- timeDeletePreviousVersions.stop();
- logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
- entityId.getUuid(), e );
- }
- } );
-
- timeDeletePreviousVersionFuture.stop();
-
- return response;
- }
+// @Override
+// public ListenableActionFuture deleteAllVersionsOfEntity(final Id entityId ) {
+// String idString = IndexingUtils.idString(entityId).toLowerCase();
+//
+// final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
+//
+// //Added For Graphite Metrics
+// final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
+// final Timer.Context timeDeleteAllVersionsFuture = allVersionsTimerFuture.time();
+//
+// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
+// .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
+//
+// response.addListener( new ActionListener<DeleteByQueryResponse>() {
+//
+// @Override
+// public void onResponse( DeleteByQueryResponse response) {
+// timeDeleteAllVersions.stop();
+// logger
+// .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+// entityId.getUuid(), response.status().toString() );
+//
+// checkDeleteByQueryResponse(tqb, response);
+// }
+//
+//
+// @Override
+// public void onFailure( Throwable e ) {
+// timeDeleteAllVersions.stop();
+// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+// entityId.getUuid(), e);
+//
+//
+// }
+// });
+// timeDeleteAllVersionsFuture.stop();
+// return response;
+// }
+//
+//
+// @Override
+// public ListenableActionFuture deletePreviousVersions(final Id entityId, final UUID version) {
+//
+// String idString = IndexingUtils.idString( entityId ).toLowerCase();
+//
+// final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
+// QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString),
+// FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
+// );
+//
+// //Added For Graphite Metrics
+// //Checks the time from the execute to the response below
+// final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
+// final Timer.Context timeDeletePreviousVersionFuture = deletePreviousTimerFuture.time();
+// final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
+// .prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
+//
+// //Added For Graphite Metrics
+// response.addListener(new ActionListener<DeleteByQueryResponse>() {
+// @Override
+// public void onResponse(DeleteByQueryResponse response) {
+// timeDeletePreviousVersions.stop();
+// //error message needs to be retooled so that it describes the entity more throughly
+// logger
+// .debug("Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
+// entityId.getType(), entityId.getUuid(), version, response.status().toString());
+//
+// checkDeleteByQueryResponse( fqb, response );
+// }
+//
+//
+// @Override
+// public void onFailure( Throwable e ) {
+// timeDeletePreviousVersions.stop();
+// logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+// entityId.getUuid(), e );
+// }
+// } );
+//
+// timeDeletePreviousVersionFuture.stop();
+//
+// return response;
+// }
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e58597be/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 3fc3e77..2762c18 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -72,8 +72,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
//the actively running subscription
private Subscription subscription;
- private Observable<List<IndexOperationMessage>> consumer;
-
private Object mutex = new Object();
@Inject
@@ -203,6 +201,9 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ indexSizeCounter.inc(indexOperationMessage.getDeIndexRequests().size());
+ indexSizeCounter.inc(indexOperationMessage.getIndexRequests().size());
+
return Observable.merge( index, deIndex );
}
} )
[07/50] incubator-usergrid git commit: graphtimers
Posted by gr...@apache.org.
graphtimers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/56c57018
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/56c57018
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/56c57018
Branch: refs/heads/USERGRID-460
Commit: 56c57018d3b4e8eb391e59bbecffb09fb6c1f1ea
Parents: b3e42dd
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 08:36:23 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 08:36:23 2015 -0600
----------------------------------------------------------------------
stack/corepersistence/graph/pom.xml | 28 +-
.../graph/impl/GraphManagerImpl.java | 288 +++++++++++++++++--
stack/pom.xml | 1 -
3 files changed, 273 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/corepersistence/graph/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/pom.xml b/stack/corepersistence/graph/pom.xml
index e448e74..b4cb45f 100644
--- a/stack/corepersistence/graph/pom.xml
+++ b/stack/corepersistence/graph/pom.xml
@@ -66,13 +66,17 @@
</dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>3.0.2</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -88,16 +92,16 @@
<build>
<plugins>
-
+
<!-- <plugin>
<groupId>org.safehaus.chop</groupId>
<artifactId>chop-maven-plugin</artifactId>
<version>${chop.version}</version>
-
+
NOTE: you should be putting most of these variables into your settings.xml
as an automatically activated profile.
-
+
<configuration>
<accessKey>${aws.s3.key}</accessKey>
@@ -119,11 +123,11 @@
<runnerCount>6</runnerCount>
<runnerName>${runner.name}</runnerName>
<securityGroupExceptions>
-
+
Add your own IP address as an exception to allow access
but please do this in the settings.xml file .. essentially
all parameters should be in the settings.xml file.
-
+
<param>${myip.address}/32:24981</param>
<param>${myip.address}/32:22</param>
</securityGroupExceptions>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index df10816..53e116d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -25,6 +25,9 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +59,12 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.MutationBatch;
+import rx.Notification;
import rx.Observable;
import rx.Observer;
import rx.Subscriber;
+import rx.functions.Action0;
+import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -81,6 +87,30 @@ public class GraphManagerImpl implements GraphManager {
private final EdgeDeleteListener edgeDeleteListener;
private final NodeDeleteListener nodeDeleteListener;
+ private final Timer writeEdgeTimer;
+ private final Meter writeEdgeMeter;
+ private final Meter deleteEdgeMeter;
+ private final Timer deleteEdgeTimer;
+ private final Meter deleteNodeMeter;
+ private final Timer deleteNodeTimer;
+ private final Meter loadEdgesFromSourceMeter;
+ private final Timer loadEdgesFromSourceTimer;
+ private final Meter loadEdgesToTargetMeter;
+ private final Timer loadEdgesToTargetTimer;
+ private final Meter loadEdgesVersionsMeter;
+ private final Timer loadEdgesVersionsTimer;
+ private final Meter loadEdgesFromSourceByTypeMeter;
+ private final Timer loadEdgesFromSourceByTypeTimer;
+ private final Meter loadEdgesToTargetByTypeMeter;
+ private final Timer loadEdgesToTargetByTypeTimer;
+ private final Timer getEdgeTypesFromSourceTimer;
+ private final Meter getEdgeTypesFromSourceMeter;
+ private final Timer getIdTypesFromSourceTimer;
+ private final Meter getIdTypesFromSourceMeter;
+ private final Meter getEdgeTypesToTargetMeter;
+ private final Timer getEdgeTypesToTargetTimer;
+ private final Timer getIdTypesToTargetTimer;
+ private final Meter getIdTypesToTargetMeter;
private Observer<Integer> edgeDeleteSubcriber;
private Observer<Integer> nodeDelete;
@@ -96,7 +126,8 @@ public class GraphManagerImpl implements GraphManager {
final GraphFig graphFig,
final EdgeDeleteListener edgeDeleteListener,
final NodeDeleteListener nodeDeleteListener,
- @Assisted final ApplicationScope scope) {
+ @Assisted final ApplicationScope scope,
+ MetricsFactory metricsFactory) {
ValidationUtils.validateApplicationScope( scope );
@@ -117,6 +148,36 @@ public class GraphManagerImpl implements GraphManager {
this.edgeDeleteSubcriber = MetricSubscriber.INSTANCE;
this.nodeDelete = MetricSubscriber.INSTANCE;
+ this.writeEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+ this.writeEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "write.edge.timer");
+ this.deleteEdgeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.edge.meter");
+ this.deleteEdgeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.edge.timer");
+ this.deleteNodeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "delete.node.meter");
+ this.deleteNodeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "delete.node.timer");
+ this.loadEdgesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.meter");
+ this.loadEdgesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.timer");
+ this.loadEdgesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.meter");
+ this.loadEdgesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.timer");
+ this.loadEdgesVersionsMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.versions.meter");
+ this.loadEdgesVersionsTimer = metricsFactory.getTimer(GraphManagerImpl.class,"load.versions.timer");
+ this.loadEdgesFromSourceByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.from.type.meter");
+ this.loadEdgesFromSourceByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.from.type.timer");
+ this.loadEdgesToTargetByTypeMeter = metricsFactory.getMeter(GraphManagerImpl.class, "load.to.type.meter");
+ this.loadEdgesToTargetByTypeTimer = metricsFactory.getTimer(GraphManagerImpl.class, "load.to.type.timer");
+
+ this.getEdgeTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.from.timer");
+ this.getEdgeTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "write.edge.meter");
+
+ this.getIdTypesFromSourceTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.idtype.from.timer");
+ this.getIdTypesFromSourceMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.from.meter");
+
+ this.getEdgeTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class,"get.edge.to.timer");
+ this.getEdgeTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.edge.to.meter");
+
+ this.getIdTypesToTargetTimer = metricsFactory.getTimer(GraphManagerImpl.class, "get.idtype.to.timer");
+ this.getIdTypesToTargetMeter = metricsFactory.getMeter(GraphManagerImpl.class, "get.idtype.to.meter");
+
+
}
@@ -125,6 +186,8 @@ public class GraphManagerImpl implements GraphManager {
GraphValidation.validateEdge( edge );
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false );
+ final Timer.Context timer = writeEdgeTimer.time();
+ final Meter meter = writeEdgeMeter;
return Observable.from( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
@Override
@@ -143,45 +206,72 @@ public class GraphManagerImpl implements GraphManager {
return edge;
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> deleteEdge( final Edge edge ) {
- GraphValidation.validateEdge( edge );
+ GraphValidation.validateEdge(edge);
- final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
+ final MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
-
- return Observable.from( markedEdge ).map( new Func1<MarkedEdge, Edge>() {
+ final Timer.Context timer = deleteEdgeTimer.time();
+ final Meter meter = deleteEdgeMeter;
+ return Observable.from(markedEdge).map(new Func1<MarkedEdge, Edge>() {
@Override
- public Edge call( final MarkedEdge edge ) {
+ public Edge call(final MarkedEdge edge) {
final UUID timestamp = UUIDGenerator.newTimeUUID();
- final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge( scope, edge, timestamp );
+ final MutationBatch edgeMutation = storageEdgeSerialization.writeEdge(scope, edge, timestamp);
- LOG.debug( "Marking edge {} as deleted to commit log", edge );
- HystrixCassandra.user( edgeMutation );
+ LOG.debug("Marking edge {} as deleted to commit log", edge);
+ HystrixCassandra.user(edgeMutation);
//HystrixCassandra.async( edgeDeleteListener.receive( scope, markedEdge,
// timestamp )).subscribeOn( Schedulers.io() ).subscribe( edgeDeleteSubcriber );
- edgeDeleteListener.receive( scope, markedEdge, timestamp ).subscribeOn( Schedulers.io() )
- .subscribe( edgeDeleteSubcriber );
+ edgeDeleteListener.receive(scope, markedEdge, timestamp).subscribeOn(Schedulers.io())
+ .subscribe(edgeDeleteSubcriber);
return edge;
}
- } );
+ })
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Id> deleteNode( final Id node, final long timestamp ) {
+ final Timer.Context timer = deleteNodeTimer.time();
+ final Meter meter = deleteNodeMeter;
return Observable.from( node ).map( new Func1<Id, Id>() {
@Override
public Id call( final Id id ) {
@@ -205,114 +295,250 @@ public class GraphManagerImpl implements GraphManager {
return id;
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super Id>>() {
+ @Override
+ public void call(Notification<? super Id> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgeVersions( final SearchByEdge searchByEdge ) {
+ final Timer.Context timer = loadEdgesVersionsTimer.time();
+ final Meter meter = loadEdgesVersionsMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( searchByEdge.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(searchByEdge.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
+ final Timer.Context timer = loadEdgesFromSourceTimer.time();
+ final Meter meter = loadEdgesFromSourceMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
+ final Timer.Context timer = loadEdgesToTargetTimer.time();
+ final Meter meter = loadEdgesToTargetMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
+ final Timer.Context timer = loadEdgesFromSourceByTypeTimer.time();
+ final Meter meter = loadEdgesFromSourceByTypeMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
-
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
+ final Timer.Context timer = loadEdgesToTargetByTypeTimer.time();
+ final Meter meter = loadEdgesToTargetByTypeMeter;
return Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
}
- } ).buffer( graphFig.getScanPageSize() ).flatMap( new EdgeBufferFilter( search.getMaxTimestamp() ) )
- .cast( Edge.class );
+ } ).buffer( graphFig.getScanPageSize() ).flatMap(new EdgeBufferFilter(search.getMaxTimestamp()))
+ .cast(Edge.class)
+ .doOnEach(new Action1<Notification<? super Edge>>() {
+ @Override
+ public void call(Notification<? super Edge> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getEdgeTypesFromSource( final SearchEdgeType search ) {
-
+ final Timer.Context timer = getEdgeTypesFromSourceTimer.time();
+ final Meter meter = getEdgeTypesFromSourceMeter;
return Observable.create( new ObservableIterator<String>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesFromSource( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getIdTypesFromSource( final SearchIdType search ) {
+ final Timer.Context timer = getIdTypesFromSourceTimer.time();
+ final Meter meter = getIdTypesFromSourceMeter;
return Observable.create( new ObservableIterator<String>( "getIdTypesFromSource" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesFromSource( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getEdgeTypesToTarget( final SearchEdgeType search ) {
-
+ final Timer.Context timer = getEdgeTypesToTargetTimer.time();
+ final Meter meter = getEdgeTypesToTargetMeter;
return Observable.create( new ObservableIterator<String>( "getEdgeTypesToTarget" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getEdgeTypesToTarget( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
@Override
public Observable<String> getIdTypesToTarget( final SearchIdType search ) {
+ final Timer.Context timer = getIdTypesToTargetTimer.time();
+ final Meter meter = getIdTypesToTargetMeter;
return Observable.create( new ObservableIterator<String>( "getIdTypesToTarget" ) {
@Override
protected Iterator<String> getIterator() {
return edgeMetadataSerialization.getIdTypesToTarget( scope, search );
}
- } );
+ } )
+ .doOnEach(new Action1<Notification<? super String>>() {
+ @Override
+ public void call(Notification<? super String> notification) {
+ meter.mark();
+ }
+ })
+ .doOnCompleted(new Action0() {
+ @Override
+ public void call() {
+ timer.stop();
+ }
+ });
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56c57018/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 25b2258..73f2cbd 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -109,7 +109,6 @@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
[28/50] incubator-usergrid git commit: Fixes worker issue in cloud
formation
Posted by gr...@apache.org.
Fixes worker issue in cloud formation
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c319d07d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c319d07d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c319d07d
Branch: refs/heads/USERGRID-460
Commit: c319d07d8c09d2e5988f58faad1dd42cff85cbe8
Parents: 9aec790
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 19:33:54 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 19:33:54 2015 -0600
----------------------------------------------------------------------
.../src/main/groovy/configure_usergrid.groovy | 8 +-
stack/awscluster/ugcluster-cf.json | 150 ++-----------------
2 files changed, 23 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 8af5884..fb98368 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -51,6 +51,9 @@ def esShards = numEsNodes*2;
def esReplicas = 1;
def tomcatThreads = System.getenv().get("TOMCAT_THREADS")
+
+def workerCount = System.getenv().get("INDEX_WORKER_COUNT")
+
//temporarily set to equal since we now have a sane tomcat thread calculation
def hystrixThreads = tomcatThreads
@@ -58,7 +61,6 @@ def hystrixThreads = tomcatThreads
def ec2Region = System.getenv().get("EC2_REGION")
def cassEc2Region = ec2Region.replace("-1", "")
-
NodeRegistry registry = new NodeRegistry();
def selectResult = registry.searchNode('cassandra')
@@ -195,6 +197,10 @@ usergrid.queue.region=${ec2Region}
usergrid.scheduler.enabled=true
usergrid.scheduler.job.workers=1
+
+#Set our ingest rate
+elasticsearch.worker_count=${workerCount}
+
"""
println usergridConfig
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 0c2a9d2..e9db671 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -46,6 +46,12 @@
],
"ConstraintDescription": "must be valid instance type."
},
+ "RestIndexWorkers":{
+ "Description": "The number of index workers to ingest ElasticSearch batch operations per tomcat",
+ "Type": "Number",
+ "Default": "8",
+ "MinValue": "3"
+ },
"TomcatThreadsPerCore": {
"Description": "Number of threads to configure tomcat for per core",
"Type": "Number",
@@ -289,7 +295,7 @@
}
}
},
- "GraphiteAutoScalingLaunchConfiguration":{
+ "GraphiteAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -464,7 +470,7 @@
}
}
},
- "OpsCenterUser": {
+ "OpsCenterUser": {
"Type": "AWS::IAM::User",
"Properties": {
"Path": "/",
@@ -485,7 +491,7 @@
]
}
},
- "OpsCenterKey": {
+ "OpsCenterKey": {
"Type": "AWS::IAM::AccessKey",
"Properties": {
"UserName": {
@@ -493,7 +499,7 @@
}
}
},
- "OpsCenterAutoScalingLaunchConfiguration":{
+ "OpsCenterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -619,7 +625,7 @@
}
},
- "OpsCenterAutoScalingGroup": {
+ "OpsCenterAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2014-07-24",
"Properties": {
@@ -767,7 +773,7 @@
}
}
},
- "CassAutoScalingLaunchConfiguration":{
+ "CassAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -955,7 +961,7 @@
}
}
},
- "ESMasterAutoScalingLaunchConfiguration":{
+ "ESMasterAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -1137,131 +1143,6 @@
}
}
},
- "RestAutoScalingLaunchConfiguration":{
- "Type":"AWS::AutoScaling::LaunchConfiguration",
- "Properties":{
- "UserData":{
- "Fn::Base64":{
- "Fn::Join":[
- "",
- [
- "#!/bin/bash -ex\n",
- "# REST SERVER STARTUP \n",
- "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
- "\n",
- "mkdir -p /usr/share/usergrid\n",
- "\n",
- "# create script that sets our environment variables\n",
- "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
- "alias sudo='sudo -E'\n",
- "\n",
- "export TYPE=rest\n",
- "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n",
- "export YOURKIT=", { "Ref":"InstallYourkit" }, "\n",
- "export DNS_NAME=", { "Ref":"DnsSubDomain" }, "\n",
- "export DNS_DOMAIN=", { "Ref":"DnsDomain" }, "\n",
- "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
- "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
- "export ELB_NAME=", { "Ref":"RestElasticLoadBalancer" }, "\n",
- "\n",
- "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
- "export EC2_REGION=", { "Ref":"AWS::Region" }, "\n",
- "export EC2_URL=https://ec2.amazonaws.com/\n", "\n",
- "export REST_SECURITY_GROUP_NAME=", { "Ref":"RestSecurityGroup" }, "\n",
- "export DB_SECURITY_GROUP_NAME=", { "Ref":"CassSecurityGroup" }, "\n",
- "\n",
- "export CASSANDRA_CLUSTER_NAME=", { "Ref":"CassClusterName" }, "\n",
- "export CASSANDRA_KEYSPACE_NAME=usergrid", "\n",
- "export CASSANDRA_NUM_SERVERS=", { "Ref":"CassNumServers" }, "\n",
- "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
- "export TOMCAT_NUM_SERVERS=", { "Ref":"RestMinServers" }, "\n",
- "\n",
- "export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor" }, "\n",
- "\n",
- "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency" }, "\n",
- "\n",
- "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
- "\n",
-
- "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
- "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
- "\n",
- "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
- "\n",
- "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
- "\n",
- "export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
- "export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
- "\n",
- "EOF\n",
- "\n",
- "# put AWS creds in environment\n",
- "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
- "export AWS_ACCESS_KEY=", { "Ref":"RestKey" }, "\n",
- "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
- "EOF\n",
- "\n",
- "# setup s3cmd (will be installed by init script) \n",
- "cat >/etc/s3cfg <<EOF\n",
- "access_key=", { "Ref":"RestKey" }, "\n",
- "secret_key=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] }, "\n",
- "EOF\n",
- "chmod 644 /etc/s3cfg\n",
- "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
- "ln -s /etc/s3cfg ~root/.s3cfg\n",
- "\n",
- "# download usergrid and init script bundle from S3\n",
- "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key | apt-key add -\n",
- "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
- "apt-get update\n",
- "apt-get -y install s3cmd\n",
- "cd /usr/share/usergrid\n",
- "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"}, "/ROOT.war\n",
- "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
- "mv ROOT.war webapps/ROOT.war\n",
- "chmod 755 ./init_instance/*.sh\n",
- "cd ./init_instance\n",
- "# Init as a REST intance \n",
- "sh ./init_rest_server.sh\n"
- ]
- ]
- }
- },
- "KeyName":{
- "Ref":"KeyPair"
- },
- "ImageId":{
- "Fn::FindInMap":[
- "AWSRegionArch2AMI",
- {
- "Ref":"AWS::Region"
- },
- {
- "Fn::FindInMap":[
- "AWSInstanceType2Arch",
- {
- "Ref":"RestInstanceType"
- },
- "Arch"
- ]
- }
- ]
- },
- "InstanceType":{
- "Ref":"RestInstanceType"
- },
- "IamInstanceProfile":{
- "Ref":"RootInstanceProfile"
- },
- "SecurityGroups":[
- {
- "Ref":"RestSecurityGroup"
- }
- ]
- }
- },
"ESAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
@@ -1378,7 +1259,7 @@
]
}
},
- "ESAutoScalingGroup": {
+ "ESAutoScalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Version": "2009-05-15",
"Properties": {
@@ -1442,7 +1323,7 @@
}
}
},
- "RestAutoScalingLaunchConfiguration":{
+ "RestAutoScalingLaunchConfiguration":{
"Type":"AWS::AutoScaling::LaunchConfiguration",
"Properties":{
"UserData":{
@@ -1487,6 +1368,7 @@
"\n",
"export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency" }, "\n",
"\n",
+ "export INDEX_WORKER_COUNT=", { "Ref":"RestIndexWorkers" }, "\n",
"export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
"export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
[43/50] incubator-usergrid git commit: Fixes bug in writeunique
verify failover command
Posted by gr...@apache.org.
Fixes bug in writeunique verify failover command
Fixes NoAWSCredsRule to use Assumptions to mark as ignored
Adds AWS creds rule to NotificationsServiceTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/03341191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/03341191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/03341191
Branch: refs/heads/USERGRID-460
Commit: 03341191a2109b8d297387c37b88df7af7b50218
Parents: f20dfd3
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 15:34:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 15:34:45 2015 -0600
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 3 ++-
.../impl/EntityVersionCleanupTask.java | 7 +++--
.../mvcc/stage/write/WriteUniqueVerify.java | 24 +++++++++--------
.../collection/util/EntityUtils.java | 28 +++++++++++++++++---
.../index/impl/EsIndexBufferConsumerImpl.java | 14 ++++++++--
.../persistence/queue/NoAWSCredsRule.java | 17 ++++++++++++
stack/services/pom.xml | 13 ++++++++-
.../notifications/NotifiersServiceIT.java | 6 +++++
8 files changed, 90 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ac04dcc..4bde50e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -392,9 +392,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
+ queryCollectionEm("dogs", "select *");
Thread.sleep(100);
crs = queryCollectionCp("dogs", "dog", "select *");
- } while ( crs.size() == numEntities && count++ < 15 );
+ } while ( crs.size() != numEntities && count++ < 15 );
Assert.assertEquals("Expect candidates without earlier stale entities", crs.size(), numEntities);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 03a5ff6..65506ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
@@ -158,10 +159,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
final Entity entity = mvccEntity.getEntity().get();
//remove all unique fields from the index
- for ( final Field field : entity.getFields() ) {
- if ( !field.isUnique() ) {
- continue;
- }
+ for ( final Field field : EntityUtils.getUniqueFields(entity )) {
+
final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
final MutationBatch deleteMutation =
uniqueValueSerializationStrategy.delete( scope, unique );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 173efa7..5bdf3b9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -31,6 +31,8 @@ import com.netflix.hystrix.HystrixCommand;
import com.netflix.hystrix.HystrixCommandGroupKey;
import com.netflix.hystrix.HystrixThreadPoolProperties;
import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.slf4j.Logger;
@@ -108,20 +110,21 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
//
// Construct all the functions for verifying we're unique
//
- for ( final Field field : entity.getFields() ) {
+
+
+ for ( final Field field : EntityUtils.getUniqueFields(entity)) {
// if it's unique, create a function to validate it and add it to the list of
// concurrent validations
- if ( field.isUnique() ) {
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- batch.mergeShallow( mb );
- uniqueFields.add(field);
- }
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+
+ batch.mergeShallow( mb );
+ uniqueFields.add(field);
}
//short circuit nothing to do
@@ -175,12 +178,11 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
- Collection<Field> entityFields = entity.getFields();
//allocate our max size, worst case
//now get the set of fields back
final UniqueValueSet uniqueValues;
try {
- uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, entityFields );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index 6233075..cf964a3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@ -1,8 +1,11 @@
package org.apache.usergrid.persistence.collection.util;
-import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.commons.lang3.reflect.FieldUtils;
@@ -16,9 +19,9 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class EntityUtils {
- private static final Field VERSION = FieldUtils.getField( Entity.class, "version", true );
+ private static final java.lang.reflect.Field VERSION = FieldUtils.getField( Entity.class, "version", true );
- private static final Field ID = FieldUtils.getField( Entity.class, "id", true );
+ private static final java.lang.reflect.Field ID = FieldUtils.getField( Entity.class, "id", true );
/**
@@ -46,4 +49,23 @@ public class EntityUtils {
throw new RuntimeException( "Unable to set the field " + ID + " into the entity", e );
}
}
+
+
+ /**
+ * Get the unique fields for an entity
+ * @param entity
+ * @return
+ */
+ public static List<Field> getUniqueFields( final Entity entity ){
+
+ final List<Field> uniqueFields = new ArrayList<>(entity.getFields().size());
+
+ for(final Field field: entity.getFields()){
+ if(field.isUnique()){
+ uniqueFields.add( field);
+ }
+ }
+
+ return uniqueFields;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index d55073a..b31cf39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -322,8 +322,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
for (BulkItemResponse response : responses) {
if (response.isFailed()) {
- throw new RuntimeException("Unable to index documents. Errors are :"
- + response.getFailure().getMessage());
+
+ final BulkItemResponse.Failure failure = response.getFailure();
+
+ final String message;
+
+ if(failure != null) {
+ message = "Unable to index documents. Errors are :" + response.getFailure().getMessage();
+ }else{
+ message = "Unable to index documents. Response is :" + response.getResponse();
+ }
+
+ throw new RuntimeException(message);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
index 45218c2..ba0dc1f 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.persistence.queue;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
@@ -49,6 +51,9 @@ public class NoAWSCredsRule implements TestRule {
throw t;
}
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
}
}
@@ -69,6 +74,18 @@ public class NoAWSCredsRule implements TestRule {
}
}
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
final Throwable cause = t.getCause();
if ( cause == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index f5e8dd5..68f1a4a 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -133,6 +133,7 @@
<scope>runtime</scope>
</dependency>
+
<dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-core</artifactId>
@@ -290,7 +291,15 @@
<classifier>tests</classifier>
</dependency>
- <dependency>
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-config</artifactId>
<version>${project.version}</version>
@@ -311,6 +320,8 @@
<scope>test</scope>
</dependency>
+
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index 3c1c645..e6970b5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -17,12 +17,15 @@
package org.apache.usergrid.services.notifications;
import org.apache.commons.io.IOUtils;
+
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
import org.apache.usergrid.persistence.entities.Notifier;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.apache.usergrid.services.notifications.ConnectionException;
import org.apache.usergrid.services.notifications.NotificationsService;
@@ -48,6 +51,9 @@ public class NotifiersServiceIT extends AbstractServiceIT {
private QueueListener listener;
+ @Rule
+ public NoAWSCredsRule noCredsRule = new NoAWSCredsRule();
+
@Before
public void before() throws Exception {
[21/50] incubator-usergrid git commit: resolve futures via done method
Posted by gr...@apache.org.
resolve futures via done method
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ad33ecfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ad33ecfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ad33ecfd
Branch: refs/heads/USERGRID-460
Commit: ad33ecfd1146a98009c141f1ea6948b9d280f7d7
Parents: 9630fcf
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 16:26:05 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 16:26:05 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexOperationMessage.java | 4 ++++
.../usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java | 1 +
.../usergrid/persistence/index/impl/BufferQueueSQSImpl.java | 2 +-
3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index a7388d6..33b68cd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -126,4 +126,8 @@ public class IndexOperationMessage implements Serializable {
result = 31 * result + deIndexRequests.hashCode();
return result;
}
+
+ public void done() {
+ getFuture().done();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index 1973e5d..6716fd1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -47,6 +47,7 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
messages.offer( operation );
+ operation.done();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ad33ecfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 25c2ba6..1b0529f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -141,7 +141,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//signal to SQS
this.queue.sendMessage( identifier );
- operation.getFuture().run();
+ operation.done();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
[26/50] incubator-usergrid git commit: change metrics prefix
Posted by gr...@apache.org.
change metrics prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e8dc17de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e8dc17de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e8dc17de
Branch: refs/heads/USERGRID-460
Commit: e8dc17de71638ec6f62412a173cc3e161046c4c2
Parents: d81dfaa
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 11 17:59:07 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 11 17:59:07 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/core/metrics/MetricsFactoryImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e8dc17de/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 773828d..6d0881b 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -49,7 +49,7 @@ public class MetricsFactoryImpl implements MetricsFactory {
if(!metricsHost.equals("false")) {
Graphite graphite = new Graphite(new InetSocketAddress(metricsHost, 2003));
graphiteReporter = GraphiteReporter.forRegistry(registry)
- .prefixedWith("notifications")
+ .prefixedWith("usergrid-metrics")
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(MetricFilter.ALL)
[18/50] incubator-usergrid git commit: Fixes issue with cluster
prefix configuration
Posted by gr...@apache.org.
Fixes issue with cluster prefix configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aeb15e96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aeb15e96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aeb15e96
Branch: refs/heads/USERGRID-460
Commit: aeb15e96a9a98909c86bbc7e0ad6ba794a21ab1c
Parents: 8fa5c9f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 11:56:52 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 11:56:52 2015 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aeb15e96/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 1f7140c..9274430 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -108,7 +108,7 @@ usergrid.write.cl=${writeConsistencyLevel}
elasticsearch.cluster_name=${clusterName}
-elasticsearch.index_prefix=usergrid
+elasticsearch.index_prefix=${stackName}
elasticsearch.hosts=${esnodes}
elasticsearch.port=9300
elasticsearch.number_shards=${esShards}
[29/50] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by gr...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/19c6ad0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/19c6ad0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/19c6ad0b
Branch: refs/heads/USERGRID-460
Commit: 19c6ad0bc01d79719079985b4930956c1ed17b48
Parents: c319d07 e8dc17d
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 19:34:13 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 19:34:13 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/core/metrics/MetricsFactoryImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[04/50] incubator-usergrid git commit: Changed consumer structure of
buffer/wait timeout. This how happens implicitly in our queue take,
and is no longer necessary. .
Posted by gr...@apache.org.
Changed consumer structure of buffer/wait timeout. This how happens implicitly in our queue take, and is no longer necessary.
.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9111d944
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9111d944
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9111d944
Branch: refs/heads/USERGRID-460
Commit: 9111d94481bc15490b477f9ca48dd2565ca0e9dd
Parents: c47f32a
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 10 17:25:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 10 17:25:23 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 9 ++-
.../usergrid/persistence/index/IndexFig.java | 47 ++++++-----
.../index/IndexOperationMessage.java | 83 ++++++++++++--------
.../index/impl/BufferQueueInMemoryImpl.java | 5 +-
.../index/impl/BufferQueueSQSImpl.java | 54 +++++--------
.../persistence/index/impl/DeIndexRequest.java | 5 ++
.../index/impl/EsEntityIndexBatchImpl.java | 12 ++-
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++++----
.../index/impl/EsIndexBufferProducerImpl.java | 3 +-
.../persistence/index/impl/IndexRequest.java | 24 ++++--
.../impl/EntityConnectionIndexImplTest.java | 4 +-
.../queue/impl/SQSQueueManagerImpl.java | 8 +-
12 files changed, 174 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 8d99586..7b53d67 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -34,6 +34,8 @@ import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.index.guice.IndexModule;
+import org.apache.usergrid.persistence.index.impl.BufferQueue;
+import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.slf4j.Logger;
@@ -69,7 +71,12 @@ public class CoreModule extends AbstractModule {
install( new CommonModule());
install(new CollectionModule());
install(new GraphModule());
- install(new IndexModule());
+ install( new IndexModule() {
+ @Override
+ public void wireBufferQueue() {
+ bind(BufferQueue.class).to( BufferQueueSQSImpl.class );
+ }
+ } );
install(new MapModule());
install(new QueueModule());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index ce14449..cde86fd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -18,6 +18,7 @@
*/
package org.apache.usergrid.persistence.index;
+
import org.safehaus.guicyfig.Default;
import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
@@ -55,8 +56,16 @@ public interface IndexFig extends GuicyFig {
public static final String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
+ /**
+ * Amount of time to wait when reading from the queue
+ */
public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
+ /**
+ * Amount of time to wait when reading from the queue in milliseconds
+ */
+ public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
+
public static final String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
public static final String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
@@ -67,9 +76,10 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
/**
- * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple backpressure
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
*/
- public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+ public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@@ -82,7 +92,7 @@ public interface IndexFig extends GuicyFig {
int getPort();
@Default( "usergrid" )
- @Key( ELASTICSEARCH_CLUSTER_NAME)
+ @Key( ELASTICSEARCH_CLUSTER_NAME )
String getClusterName();
@Default( "usergrid" ) // no underbars allowed
@@ -111,15 +121,15 @@ public interface IndexFig extends GuicyFig {
public boolean isForcedRefresh();
/** Identify the client node with a unique name. */
- @Default("default")
+ @Default( "default" )
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
- @Default("6")
+ @Default( "6" )
@Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
public int getNumberOfShards();
- @Default("1")
+ @Default( "1" )
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
public int getNumberOfReplicas();
@@ -127,51 +137,48 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_FAIL_REFRESH )
int getFailRefreshCount();
- @Default("2")
+ @Default( "2" )
int getIndexCacheMaxWorkers();
/**
* how long to wait before the buffer flushes to send
- * @return
*/
- @Default("250")
+ @Default( "250" )
@Key( INDEX_BUFFER_TIMEOUT )
long getIndexBufferTimeout();
/**
* size of the buffer to build up before you send results
- * @return
*/
- @Default("1000")
+ @Default( "1000" )
@Key( INDEX_BUFFER_SIZE )
int getIndexBufferSize();
/**
* size of the buffer to build up before you send results
- * @return
*/
- @Default("1000")
+ @Default( "1000" )
@Key( INDEX_QUEUE_SIZE )
int getIndexQueueSize();
/**
* Request batch size for ES
- * @return
*/
- @Default("1000")
- @Key( INDEX_BATCH_SIZE)
+ @Default( "1000" )
+ @Key( INDEX_BATCH_SIZE )
int getIndexBatchSize();
- @Default("one")
+ @Default( "one" )
@Key( INDEX_WRITE_CONSISTENCY_LEVEL )
String getWriteConsistencyLevel();
- @Default("1000")
+ @Default( "1000" )
@Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
long getFailureRetryTime();
//give us 60 seconds to process the message
- @Default("60")
- @Key(INDEX_QUEUE_READ_TIMEOUT)
+ @Default( "60" )
+ @Key( INDEX_QUEUE_READ_TIMEOUT )
int getIndexQueueTimeout();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
index 43eaa01..7d8a859 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexOperationMessage.java
@@ -16,69 +16,84 @@
*/
package org.apache.usergrid.persistence.index;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.index.impl.BatchRequest;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.support.replication.ShardReplicationOperationRequestBuilder;
import java.io.Serializable;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Set;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
/**
* Container for index operations.
*/
-public class IndexOperationMessage implements Serializable {
- private final Set<BatchRequest> builders;
+public class IndexOperationMessage implements Serializable {
+ private final Set<IndexRequest> indexRequests;
+ private final Set<DeIndexRequest> deIndexRequests;
+
+
+
private final BetterFuture<IndexOperationMessage> containerFuture;
- public IndexOperationMessage(){
+
+ public IndexOperationMessage() {
final IndexOperationMessage parent = this;
- this.builders = new HashSet<>();
- this.containerFuture = new BetterFuture<>(new Callable<IndexOperationMessage>() {
+ this.indexRequests = new HashSet<>();
+ this.deIndexRequests = new HashSet<>();
+ this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
@Override
public IndexOperationMessage call() throws Exception {
return parent;
}
- });
+ } );
}
- /**
- * Add all our operations in the set
- * @param requests
- */
- public void setOperations(final Set<BatchRequest> requests){
- this.builders.addAll( requests);
+ public void addIndexRequest( final IndexRequest indexRequest ) {
+ indexRequests.add( indexRequest );
}
- /**
- * Add the operation to the set
- * @param builder
- */
- public void addOperation(BatchRequest builder){
- builders.add(builder);
+ public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
+ indexRequests.addAll( indexRequests );
}
- /**
- * return operations for the message
- * @return
- */
- public Set<BatchRequest> getOperations(){
- return builders;
+
+ public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
+ deIndexRequests.add( deIndexRequest );
+ }
+
+
+ public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
+ deIndexRequests.addAll( deIndexRequests );
+ }
+
+
+ public Set<IndexRequest> getIndexRequests() {
+ return indexRequests;
+ }
+
+
+ public Set<DeIndexRequest> getDeIndexRequests() {
+ return deIndexRequests;
+ }
+
+
+ @JsonIgnore
+ public boolean isEmpty(){
+ return indexRequests.isEmpty() && deIndexRequests.isEmpty();
}
/**
* return the promise
- * @return
*/
- public BetterFuture<IndexOperationMessage> getFuture(){
+ @JsonIgnore
+ public BetterFuture<IndexOperationMessage> getFuture() {
return containerFuture;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
index ef0ef5f..1973e5d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
@@ -60,8 +60,11 @@ public class BufferQueueInMemoryImpl implements BufferQueue {
//loop until we're we're full or we time out
do {
try {
+
+ final long remaining = endTime - System.currentTimeMillis();
+
//we received 1, try to drain
- IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+ IndexOperationMessage polled = messages.poll( remaining, timeUnit );
//drain
if ( polled != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index b814603..833e045 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -21,15 +21,11 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import org.elasticsearch.action.ActionRequestBuilder;
-
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -39,7 +35,6 @@ import org.apache.usergrid.persistence.queue.QueueMessage;
import org.apache.usergrid.persistence.queue.QueueScope;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -69,13 +64,10 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
- final Message toQueue = new Message( operation.getOperations() );
-
-
try {
- this.queue.sendMessage( toQueue );
+ this.queue.sendMessage( operation );
operation.getFuture().run();
}
catch ( IOException e ) {
@@ -87,19 +79,22 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
- //loop until we're we're full or we time out
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
List<QueueMessage> messages = queue
- .getMessages( takeSize, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- Message.class );
+ .getMessages( actualTake, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ IndexOperationMessage.class );
final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
for ( final QueueMessage message : messages ) {
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message );
+ final IndexOperationMessage messageBody = ( IndexOperationMessage ) message.getBody();
- operation.setOperations( ( ( Message ) message.getBody() ).getData() );
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage(message, messageBody );
response.add( operation );
}
@@ -111,10 +106,15 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public void ack( final List<IndexOperationMessage> messages ) {
+ //nothing to do
+ if(messages.size() == 0){
+ return;
+ }
+
List<QueueMessage> toAck = new ArrayList<>( messages.size() );
- for(IndexOperationMessage ioe: messages){
- toAck.add( ((SqsIndexOperationMessage)ioe).getMessage() );
+ for ( IndexOperationMessage ioe : messages ) {
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
}
queue.commitMessages( toAck );
@@ -122,22 +122,6 @@ public class BufferQueueSQSImpl implements BufferQueue {
/**
- * The message to queue to SQS
- */
- public static final class Message implements Serializable {
- private final Set<BatchRequest> data;
-
-
- private Message( final Set<BatchRequest> data ) {this.data = data;}
-
-
- public Set<BatchRequest> getData() {
- return data;
- }
- }
-
-
- /**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
*/
public class SqsIndexOperationMessage extends IndexOperationMessage {
@@ -145,7 +129,11 @@ public class BufferQueueSQSImpl implements BufferQueue {
private final QueueMessage message;
- public SqsIndexOperationMessage( final QueueMessage message ) {this.message = message;}
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexOperationMessage source ) {
+ this.message = message;
+ this.addAllDeIndexRequest( source.getDeIndexRequests() );
+ this.addAllIndexRequest( source.getIndexRequests() );
+ }
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index a279f16..c63c4df 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -26,10 +26,15 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.client.Client;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
/**
* Represent the properties required to build an index request
*/
+@JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class")
public class DeIndexRequest implements BatchRequest {
public final String[] indexes;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b0c731e..b63dfe6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -127,7 +127,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
final String entityType = entity.getId().getType();
- container.addOperation(new IndexRequest(alias.getWriteAlias(), entityType, indexId, entityAsMap));
+ container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType, indexId, entityAsMap));
return this;
}
@@ -168,7 +168,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
indexes = new String[]{indexIdentifier.getIndex(null)};
}
- container.addOperation( new DeIndexRequest( indexes, entityType, indexId ) );
+ container.addDeIndexRequest( new DeIndexRequest( indexes, entityType, indexId ) );
log.debug("Deindexed Entity with index id " + indexId);
@@ -192,6 +192,14 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
public BetterFuture execute() {
IndexOperationMessage tempContainer = container;
container = new IndexOperationMessage();
+
+ /**
+ * No-op, just disregard it
+ */
+ if(tempContainer.isEmpty()){
+ return tempContainer.getFuture();
+ }
+
return indexBatchBufferProducer.put(tempContainer);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 2342398..8547889 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -82,9 +82,9 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
final AtomicInteger countFail = new AtomicInteger();
//batch up sets of some size and send them in batch
- this.consumer = Observable.create( new Observable.OnSubscribe<IndexOperationMessage>() {
+ this.consumer = Observable.create( new Observable.OnSubscribe<List<IndexOperationMessage>>() {
@Override
- public void call( final Subscriber<? super IndexOperationMessage> subscriber ) {
+ public void call( final Subscriber<? super List<IndexOperationMessage>> subscriber ) {
//name our thread so it's easy to see
Thread.currentThread().setName( "QueueConsumer_" + Thread.currentThread().getId() );
@@ -99,21 +99,22 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
drainList = bufferQueue
.take( config.getIndexBufferSize(), config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS );
- for ( IndexOperationMessage drained : drainList ) {
- subscriber.onNext( drained );
- }
- bufferQueue.ack( drainList );
+ subscriber.onNext( drainList );
+
timer.stop();
countFail.set( 0 );
}
- catch( EsRejectedExecutionException err) {
+ catch ( EsRejectedExecutionException err ) {
countFail.incrementAndGet();
- log.error( "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. Failed {} consecutive times", config.getFailRefreshCount(), countFail.get() );
+ log.error(
+ "Elasticsearch rejected our request, sleeping for {} milliseconds before retrying. "
+ + "Failed {} consecutive times",
+ config.getFailRefreshCount(), countFail.get() );
- //es rejected the exception, sleep and retry in the queue
+ //es rejected the exception, sleep and retry in the queue
try {
Thread.sleep( config.getFailureRetryTime() );
}
@@ -131,8 +132,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
while ( true );
}
- } ).subscribeOn( Schedulers.newThread() ).buffer( config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
- config.getIndexBufferSize() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
+ } ).subscribeOn( Schedulers.newThread() ).doOnNext( new Action1<List<IndexOperationMessage>>() {
@Override
public void call( List<IndexOperationMessage> containerList ) {
if ( containerList.size() > 0 ) {
@@ -142,7 +142,14 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
time.stop();
}
}
- } );
+} )
+ //ack after we process
+ .doOnNext( new Action1<List<IndexOperationMessage>>() {
+ @Override
+ public void call( final List<IndexOperationMessage> indexOperationMessages ) {
+ bufferQueue.ack( indexOperationMessages );
+ }
+ } );
//start in the background
consumer.subscribe();
@@ -163,15 +170,17 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
@Override
public Observable<BatchRequest> call( IndexOperationMessage operationMessage ) {
- return Observable.from( operationMessage.getOperations() );
+ final Observable<DeIndexRequest> deIndex = Observable.from( operationMessage.getDeIndexRequests () );
+ final Observable<IndexRequest> index = Observable.from( operationMessage.getIndexRequests() );
+
+ return Observable.merge( deIndex, index );
}
} );
//batch shard operations into a bulk request
- flattenMessages
- .buffer(config.getIndexBatchSize())
+ flattenMessages.toList()
.doOnNext(new Action1<List<BatchRequest>>() {
@Override
public void call(List<BatchRequest> builders) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
index db1f50e..61d5d25 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
@@ -54,7 +54,8 @@ public class EsIndexBufferProducerImpl implements IndexBufferProducer {
public BetterFuture put(IndexOperationMessage message){
Preconditions.checkNotNull(message, "Message cannot be null");
- indexSizeCounter.inc(message.getOperations().size());
+ indexSizeCounter.inc(message.getDeIndexRequests().size());
+ indexSizeCounter.inc(message.getIndexRequests().size());
Timer.Context time = timer.time();
bufferQueue.offer( message );
time.stop();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 381d005..4ec4092 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -26,17 +26,20 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
/**
* Represent the properties required to build an index request
*/
+@JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" )
public class IndexRequest implements BatchRequest {
- public final String writeAlias;
- public final String entityType;
- public final String documentId;
+ public String writeAlias;
+ public String entityType;
+ public String documentId;
- public final Map<String, Object> data;
+ public Map<String, Object> data;
public IndexRequest( final String writeAlias, final String entityType, final String documentId,
@@ -48,13 +51,18 @@ public class IndexRequest implements BatchRequest {
}
- public void doOperation(final Client client, final BulkRequestBuilder bulkRequest ){
- IndexRequestBuilder builder =
- client.prepareIndex(writeAlias, entityType, documentId).setSource( data );
+ /**
+ * DO NOT DELETE! Required for Jackson
+ */
+ public IndexRequest() {
+ }
- bulkRequest.add( builder );
+ public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
+ IndexRequestBuilder builder = client.prepareIndex( writeAlias, entityType, documentId ).setSource( data );
+
+ bulkRequest.add( builder );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 37b5e90..215ff57 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -141,7 +141,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
EsTestUtils.waitForTasks(personLikesIndex);
- Thread.sleep( 1000 );
+ Thread.sleep( 30000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex
@@ -271,7 +271,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
personLikesIndex.refresh();
EsTestUtils.waitForTasks( personLikesIndex );
- Thread.sleep( 1000 );
+ Thread.sleep( 30000 );
// now, let's search for muffins
CandidateResults likes = personLikesIndex.search( searchScope,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9111d944/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index f202fda..10aa621 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -95,7 +95,8 @@ public class SQSQueueManagerImpl implements QueueManager {
sqs.setRegion(region);
smileFactory.delegateToTextual(true);
mapper = new ObjectMapper( smileFactory );
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
+ //pretty print, disabling for speed
+// mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
} catch ( Exception e ) {
LOG.warn("failed to setup SQS",e);
@@ -180,7 +181,10 @@ public class SQSQueueManagerImpl implements QueueManager {
}
String url = getQueue().getUrl();
LOG.info("Sending Message...{} to {}",body.toString(),url);
- SendMessageRequest request = new SendMessageRequest(url,toString((Serializable)body));
+
+ final String stringBody = toString(body);
+
+ SendMessageRequest request = new SendMessageRequest(url, stringBody);
sqs.sendMessage(request);
}