You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/04/21 17:55:10 UTC
[01/16] incubator-usergrid git commit: Initial pass at moving queues
to core
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-578 f130b7cea -> a0c2651e3
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index e55fcc2..6bcc405 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.persistence.index.impl;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -33,19 +32,19 @@ import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexFig;
import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
+import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
import rx.Subscriber;
-import rx.Subscription;
import rx.schedulers.Schedulers;
@@ -54,179 +53,110 @@ import rx.schedulers.Schedulers;
*/
@Singleton
public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
- private static final Logger log = LoggerFactory.getLogger(EsIndexBufferConsumerImpl.class);
+ private static final Logger log = LoggerFactory.getLogger( EsIndexBufferConsumerImpl.class );
private final IndexFig config;
private final FailureMonitorImpl failureMonitor;
private final Client client;
private final Timer flushTimer;
- private final Counter indexSizeCounter;
private final Counter indexErrorCounter;
private final Meter flushMeter;
private final Timer produceTimer;
- private final BufferQueue bufferQueue;
private final IndexFig indexFig;
- private final AtomicLong counter = new AtomicLong( );
+ private final AtomicLong counter = new AtomicLong();
+
+
+ private final Counter indexSizeCounter;
- //the actively running subscription
- private List<Subscription> subscriptions;
+ private final Timer offerTimer;
- private Object mutex = new Object();
+ private final BufferProducer bufferProducer;
+
+
+ private AtomicLong inFlight = new AtomicLong();
- private AtomicLong inFlight = new AtomicLong( );
@Inject
- public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
- metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
+ public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider,
+ final MetricsFactory metricsFactory, final IndexFig indexFig ) {
- this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "buffer.flush");
- this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "buffer.meter");
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "buffer.size");
- this.indexErrorCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "error.count");
+ this.flushTimer = metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "buffer.flush" );
+ this.flushMeter = metricsFactory.getMeter( EsIndexBufferConsumerImpl.class, "buffer.meter" );
+ this.indexSizeCounter = metricsFactory.getCounter( EsIndexBufferConsumerImpl.class, "buffer.size" );
+ this.indexErrorCounter = metricsFactory.getCounter( EsIndexBufferConsumerImpl.class, "error.count" );
+ this.offerTimer = metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "index.buffer.producer.timer" );
//wire up the gauge of inflight messages
- metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return inFlight.longValue();
- }
- } );
-
+ metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", () -> inFlight.longValue() );
this.config = config;
- this.failureMonitor = new FailureMonitorImpl(config,provider);
+ this.failureMonitor = new FailureMonitorImpl( config, provider );
this.client = provider.getClient();
- this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
- this.bufferQueue = bufferQueue;
+ this.produceTimer =
+ metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "index.buffer.consumer.messageFetch" );
this.indexFig = indexFig;
- subscriptions = new ArrayList<>( indexFig.getWorkerCount() );
+ this.bufferProducer = new BufferProducer();
//batch up sets of some size and send them in batch
- start();
- }
+ startSubscription();
+ }
- /**
- * Loop throught and start the workers
- */
- public void start() {
- final int count = indexFig.getWorkerCount();
- for(int i = 0; i < count; i ++){
- startWorker();
- }
+ public BetterFuture put( IndexOperationMessage message ) {
+ Preconditions.checkNotNull( message, "Message cannot be null" );
+ indexSizeCounter.inc( message.getDeIndexRequests().size() );
+ indexSizeCounter.inc( message.getIndexRequests().size() );
+ Timer.Context time = offerTimer.time();
+ bufferProducer.send( message );
+ time.stop();
+ return message.getFuture();
}
/**
- * Stop the workers
+ * Start the subscription
*/
- public void stop() {
- synchronized ( mutex ) {
- //stop consuming
-
- for(final Subscription subscription: subscriptions){
- subscription.unsubscribe();
- }
- }
- }
-
-
- private void startWorker(){
- synchronized ( mutex) {
-
- Observable<List<IndexIdentifierImpl.IndexOperationMessage>> consumer = Observable.create(
- new Observable.OnSubscribe<List<IndexIdentifierImpl.IndexOperationMessage>>() {
- @Override
- public void call( final Subscriber<? super List<IndexIdentifierImpl.IndexOperationMessage>> subscriber ) {
-
- //name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
-
-
- List<IndexIdentifierImpl.IndexOperationMessage> drainList = null;
-
- do {
-
- Timer.Context timer = produceTimer.time();
+ private void startSubscription() {
- try {
+ Observable.create( bufferProducer )
+ .buffer( indexFig.getIndexBufferSize(), indexFig.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
+ Schedulers.newThread() )
+ .doOnNext( containerList -> {
+ if ( containerList.size() == 0 ) {
+ return;
+ }
- drainList = bufferQueue
- .take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
- TimeUnit.MILLISECONDS );
+ flushMeter.mark( containerList.size() );
+ Timer.Context time = flushTimer.time();
- subscriber.onNext( drainList );
+ execute( containerList );
- //take since we're in flight
- inFlight.addAndGet( drainList.size() );
-
-
- timer.stop();
- }
- //DO NOT add any doOnError* functions to this subscription. We want the producer
- //to receive these exceptions and sleep before a retry
- catch ( Throwable t ) {
- final long sleepTime = config.getFailureRetryTime();
-
- log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t );
-
- if ( drainList != null ) {
- inFlight.addAndGet( -1 * drainList.size() );
- }
-
- try {
- Thread.sleep( sleepTime );
- }
- catch ( InterruptedException ie ) {
- //swallow
- }
-
- indexErrorCounter.inc();
- }
- }
- while ( true );
- }
- } ).doOnNext( containerList -> {
- if ( containerList.size() == 0 ) {
- return;
- }
-
- flushMeter.mark(containerList.size());
- Timer.Context time = flushTimer.time();
-
-
- execute(containerList);
-
- time.stop();
- } )
+ time.stop();
+ } )
//ack after we process
- .doOnNext( indexOperationMessages -> {
- bufferQueue.ack( indexOperationMessages );
- //release so we know we've done processing
- inFlight.addAndGet( -1 * indexOperationMessages.size() );
- } ).subscribeOn( Schedulers.newThread() );
+ .doOnNext( indexOperationMessages -> {
- //start in the background
+ //release so we know we've done processing
+ inFlight.addAndGet( -1 * indexOperationMessages.size() );
+ } ).subscribe();
- final Subscription subscription = consumer.subscribe();
+ //start in the background
- subscriptions.add(subscription );
- }
}
/**
* Execute the request, check for errors, then re-init the batch for future use
*/
- private void execute( final List<IndexIdentifierImpl.IndexOperationMessage> operationMessages ) {
+ private void execute( final List<IndexOperationMessage> operationMessages ) {
if ( operationMessages == null || operationMessages.size() == 0 ) {
return;
@@ -250,28 +180,28 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.doOnNext( bulkRequestBuilder -> sendRequest( bulkRequestBuilder ) ).toBlocking().lastOrDefault( null );
//call back all futures
- Observable.from( operationMessages ).doOnNext( operationMessage -> operationMessage.getFuture().done() ).toBlocking().lastOrDefault( null );
+ Observable.from( operationMessages ).doOnNext( operationMessage -> operationMessage.getFuture().done() )
+ .toBlocking().lastOrDefault( null );
}
/**
* initialize request
- * @return
*/
private BulkRequestBuilder initRequest() {
BulkRequestBuilder bulkRequest = client.prepareBulk();
- bulkRequest.setConsistencyLevel(WriteConsistencyLevel.fromString(config.getWriteConsistencyLevel()));
- bulkRequest.setRefresh(config.isForcedRefresh());
+ bulkRequest.setConsistencyLevel( WriteConsistencyLevel.fromString( config.getWriteConsistencyLevel() ) );
+ bulkRequest.setRefresh( config.isForcedRefresh() );
return bulkRequest;
}
+
/**
* send bulk request
- * @param bulkRequest
*/
- private void sendRequest(BulkRequestBuilder bulkRequest) {
+ private void sendRequest( BulkRequestBuilder bulkRequest ) {
//nothing to do, we haven't added anything to the index
- if (bulkRequest.numberOfActions() == 0) {
+ if ( bulkRequest.numberOfActions() == 0 ) {
return;
}
@@ -280,9 +210,10 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
try {
responses = bulkRequest.execute().actionGet();
- } catch (Throwable t) {
- log.error("Unable to communicate with elasticsearch");
- failureMonitor.fail("Unable to execute batch", t);
+ }
+ catch ( Throwable t ) {
+ log.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@ -290,23 +221,43 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
boolean error = false;
- for (BulkItemResponse response : responses) {
+ for ( BulkItemResponse response : responses ) {
- if (response.isFailed()) {
+ if ( response.isFailed() ) {
// log error and continue processing
- log.error("Unable to index id={}, type={}, index={}, failureMessage={} ",
- response.getId(),
- response.getType(),
- response.getIndex(),
- response.getFailureMessage()
- );
+ log.error( "Unable to index id={}, type={}, index={}, failureMessage={} ", response.getId(),
+ response.getType(), response.getIndex(), response.getFailureMessage() );
error = true;
}
}
if ( error ) {
- throw new RuntimeException("Error during processing of bulk index operations one of the responses failed. Check previous log entries");
+ throw new RuntimeException(
+ "Error during processing of bulk index operations one of the responses failed. Check previous log "
+ + "entries" );
+ }
+ }
+
+
+ public static class BufferProducer implements Observable.OnSubscribe<IndexOperationMessage> {
+
+ private Subscriber<? super IndexOperationMessage> subscriber;
+
+
+ /**
+ * Send the data through the buffer
+ */
+ public void send( final IndexOperationMessage indexOp ) {
+
+ subscriber.onNext( indexOp );
+ }
+
+
+ @Override
+ public void call( final Subscriber<? super IndexOperationMessage> subscriber ) {
+ //just assigns for later use, doesn't do anything else
+ this.subscriber = subscriber;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
deleted file mode 100644
index 9ba92d0..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index.impl;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-
-
-/**
- * Producer for index operation messages
- */
-@Singleton
-public class EsIndexBufferProducerImpl implements IndexBufferProducer {
-
- private final Counter indexSizeCounter;
-
- private final Timer timer;
- private final BufferQueue bufferQueue;
-
- @Inject
- public EsIndexBufferProducerImpl( MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
- this.bufferQueue = bufferQueue;
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferProducerImpl.class, "index.buffer.size");
- this.timer = metricsFactory.getTimer(EsIndexBufferProducerImpl.class,"index.buffer.producer.timer");
- }
-
- public BetterFuture put(IndexIdentifierImpl.IndexOperationMessage message){
- Preconditions.checkNotNull(message, "Message cannot be null");
- indexSizeCounter.inc(message.getDeIndexRequests().size());
- indexSizeCounter.inc(message.getIndexRequests().size());
- Timer.Context time = timer.time();
- bufferQueue.offer( message );
- time.stop();
- return message.getFuture();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
index fe31d35..fed9a50 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
@@ -98,30 +98,4 @@ public class FailureMonitorImpl implements FailureMonitor {
public void success() {
failCounter.set( 0 );
}
-
-
- /**
- * Identifier for where an index is in underlying server
- */
- public static interface IndexIdentifier {
-
- /**
- * get the alias name
- * @return
- */
- IndexAlias getAlias();
-
- /**
- * get index name from suffix
- * @param suffix
- * @return
- */
- String getIndex( String suffix );
-
- /**
- * return unique string
- * @return
- */
- String toString();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
new file mode 100644
index 0000000..4b84db7
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+public class FlushBufferQueue {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
index b8e733d..3258444 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
@@ -19,6 +19,10 @@
*/
package org.apache.usergrid.persistence.index.impl;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+
+
/**
* Buffer index requests
*/
@@ -26,12 +30,9 @@ public interface IndexBufferConsumer {
/**
- * Start the consumer
- */
- void start();
-
- /**
- * Stop the consumers
+ * Put this operation into our collapsing bufer
+ * @param message
+ * @return
*/
- void stop();
+ BetterFuture put(IndexOperationMessage message);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
deleted file mode 100644
index 36cb180..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index.impl;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-
-
-/**
- * Produce to index buffer consumer
- */
-public interface IndexBufferProducer {
-
- BetterFuture put(IndexIdentifierImpl.IndexOperationMessage message);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
new file mode 100644
index 0000000..332c7b7
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+/**
+ * Identifier for where an index is in underlying server
+ */
+public interface IndexIdentifier {
+
+ /**
+ * get the alias name
+ * @return
+ */
+ IndexAlias getAlias();
+
+ /**
+ * get index name from suffix
+ * @param suffix
+ * @return
+ */
+ String getIndex( String suffix );
+
+ /**
+ * return unique string
+ * @return
+ */
+ String toString();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
index c47b2c8..782625b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
@@ -20,22 +20,15 @@
package org.apache.usergrid.persistence.index.impl;
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Callable;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.inject.Inject;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.IndexFig;
/**
* Class is used to generate an index name and alias name
*/
-public class IndexIdentifierImpl implements FailureMonitorImpl.IndexIdentifier {
+public class IndexIdentifierImpl implements IndexIdentifier {
private final IndexFig config;
@Inject
@@ -71,113 +64,4 @@ public class IndexIdentifierImpl implements FailureMonitorImpl.IndexIdentifier {
public String toString() {
return "index id"+config.getIndexPrefix();
}
-
-
- /**
- * Container for index operations.
- */
- public static class IndexOperationMessage implements Serializable {
- private final Set<IndexRequest> indexRequests;
- private final Set<DeIndexRequest> deIndexRequests;
-
-
-
- private final BetterFuture<IndexOperationMessage> containerFuture;
-
-
- public IndexOperationMessage() {
- final IndexOperationMessage parent = this;
- this.indexRequests = new HashSet<>();
- this.deIndexRequests = new HashSet<>();
- this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
- @Override
- public IndexOperationMessage call() throws Exception {
- return parent;
- }
- } );
- }
-
-
- public void addIndexRequest( final IndexRequest indexRequest ) {
- indexRequests.add( indexRequest );
- }
-
-
- public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
- this.indexRequests.addAll( indexRequests );
- }
-
-
- public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
- this.deIndexRequests.add( deIndexRequest );
- }
-
-
- public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
- this.deIndexRequests.addAll( deIndexRequests );
- }
-
-
- public Set<IndexRequest> getIndexRequests() {
- return indexRequests;
- }
-
-
- public Set<DeIndexRequest> getDeIndexRequests() {
- return deIndexRequests;
- }
-
-
- @JsonIgnore
- public boolean isEmpty(){
- return indexRequests.isEmpty() && deIndexRequests.isEmpty();
- }
-
- /**
- * return the promise
- */
- @JsonIgnore
- public BetterFuture<IndexOperationMessage> getFuture() {
- return containerFuture;
- }
-
-
- @Override
- public boolean equals( final Object o ) {
- if ( this == o ) {
- return true;
- }
- if ( o == null || getClass() != o.getClass() ) {
- return false;
- }
-
- final IndexOperationMessage that = ( IndexOperationMessage ) o;
-
- if ( !deIndexRequests.equals( that.deIndexRequests ) ) {
- return false;
- }
- if ( !indexRequests.equals( that.indexRequests ) ) {
- return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode() {
- int result = indexRequests.hashCode();
- result = 31 * result + deIndexRequests.hashCode();
- return result;
- }
-
- public void done() {
- //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
- final BetterFuture<IndexOperationMessage> future = getFuture();
-
- if(future != null ){
- future.done();
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
new file mode 100644
index 0000000..1a60026
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+
+/**
+ * Container for index operations.
+ */
+public class IndexOperationMessage implements Serializable {
+ private final Set<IndexRequest> indexRequests;
+ private final Set<DeIndexRequest> deIndexRequests;
+
+
+
+ private final BetterFuture<IndexOperationMessage> containerFuture;
+
+
+ public IndexOperationMessage() {
+ final IndexOperationMessage parent = this;
+ this.indexRequests = new HashSet<>();
+ this.deIndexRequests = new HashSet<>();
+ this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
+ @Override
+ public IndexOperationMessage call() throws Exception {
+ return parent;
+ }
+ } );
+ }
+
+
+ public void addIndexRequest( final IndexRequest indexRequest ) {
+ indexRequests.add( indexRequest );
+ }
+
+
+ public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
+ this.indexRequests.addAll( indexRequests );
+ }
+
+
+ public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
+ this.deIndexRequests.add( deIndexRequest );
+ }
+
+
+ public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
+ this.deIndexRequests.addAll( deIndexRequests );
+ }
+
+
+ public Set<IndexRequest> getIndexRequests() {
+ return indexRequests;
+ }
+
+
+ public Set<DeIndexRequest> getDeIndexRequests() {
+ return deIndexRequests;
+ }
+
+
+ @JsonIgnore
+ public boolean isEmpty(){
+ return indexRequests.isEmpty() && deIndexRequests.isEmpty();
+ }
+
+ /**
+ * return the promise
+ */
+ @JsonIgnore
+ public BetterFuture<IndexOperationMessage> getFuture() {
+ return containerFuture;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final IndexOperationMessage that = ( IndexOperationMessage ) o;
+
+ if ( !deIndexRequests.equals( that.deIndexRequests ) ) {
+ return false;
+ }
+ if ( !indexRequests.equals( that.indexRequests ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = indexRequests.hashCode();
+ result = 31 * result + deIndexRequests.hashCode();
+ return result;
+ }
+
+ public void done() {
+ //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
+ final BetterFuture<IndexOperationMessage> future = getFuture();
+
+ if(future != null ){
+ future.done();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index f3663cd..5052ddf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -61,14 +61,14 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
private final IndexAlias alias;
private final IndexCache indexCache;
private final EsProvider esProvider;
- private final IndexBufferProducer producer;
+ private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer timer;
@Inject
- public IndexRefreshCommandImpl( FailureMonitorImpl.IndexIdentifier indexIdentifier, EsProvider esProvider,
- IndexBufferProducer producer, IndexFig indexFig, MetricsFactory metricsFactory,
+ public IndexRefreshCommandImpl( IndexIdentifier indexIdentifier, EsProvider esProvider,
+ IndexBufferConsumer producer, IndexFig indexFig, MetricsFactory metricsFactory,
final IndexCache indexCache ) {
@@ -105,7 +105,7 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
IndexRequest indexRequest = new IndexRequest( alias.getWriteAlias(), docId, entityData );
//save the item
- IndexIdentifierImpl.IndexOperationMessage message = new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage message = new IndexOperationMessage();
message.addIndexRequest( indexRequest );
producer.put( message );
@@ -153,8 +153,8 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
new DeIndexRequest( aliases, appScope, edge, entity.getId(), entity.getVersion() );
//delete the item
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage =
- new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage indexOperationMessage =
+ new IndexOperationMessage();
indexOperationMessage.addDeIndexRequest( deIndexRequest );
producer.put( indexOperationMessage );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
index c60de42..c93fd86 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
@@ -20,15 +20,15 @@
package org.apache.usergrid.persistence.index.migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.impl.FailureMonitorImpl;
import org.apache.usergrid.persistence.index.impl.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
/**
* Class is used to generate an index name and alias name the old way via app name
*/
-public class LegacyIndexIdentifier implements FailureMonitorImpl.IndexIdentifier {
+public class LegacyIndexIdentifier implements IndexIdentifier {
private final IndexFig config;
private final ApplicationScope applicationScope;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 20daf82..410f0e3 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,27 +19,17 @@
package org.apache.usergrid.persistence.index.guice;
-import com.google.inject.Inject;
-import com.google.inject.TypeLiteral;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.impl.IndexBufferProducer;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.IndexCache;
-import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import rx.Observable;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import java.util.UUID;
+import com.google.inject.Inject;
+import com.google.inject.TypeLiteral;
+
+import rx.Observable;
public class TestIndexModule extends TestModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
deleted file mode 100644
index 43e581a..0000000
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchType;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.guice.TestIndexModule;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-
-import com.google.inject.Inject;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
-@NotThreadSafe
-public class BufferQueueSQSImplTest {
-
-
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
-
-
- @Rule
- public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
-
- @Inject
- public QueueManagerFactory queueManagerFactory;
-
- @Inject
- public IndexFig indexFig;
-
- @Inject
- public MapManagerFactory mapManagerFactory;
-
- @Inject
- public MetricsFactory metricsFactory;
-
-
- private BufferQueueSQSImpl bufferQueueSQS;
-
- @Before
- public void setup(){
- bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
- }
-
-
-
-
- @Test
- public void testMessageIndexing(){
-
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
- final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
- assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
-
- final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
-
-
- final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
-
-
- //de-index request
- final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
-
-
- SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
-
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
- SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
-
-
-
-
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage = new IndexIdentifierImpl.IndexOperationMessage();
- indexOperationMessage.addIndexRequest( indexRequest1);
- indexOperationMessage.addIndexRequest( indexRequest2);
-
- indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
- indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
-
- bufferQueueSQS.offer( indexOperationMessage );
-
- //wait for it to send to SQS
- indexOperationMessage.getFuture().get();
-
- //now get it back
-
- final List<IndexIdentifierImpl.IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
-
- assertTrue(ops.size() > 0);
-
- final IndexIdentifierImpl.IndexOperationMessage returnedOperation = ops.get( 0 );
-
- //get the operations out
-
- final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
-
- assertTrue(indexRequestSet.contains(indexRequest1));
- assertTrue(indexRequestSet.contains(indexRequest2));
-
-
- final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
-
- assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
- assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
-
-
-
- //now ack the message
-
- bufferQueueSQS.ack( ops );
-
- }
-
- private List<IndexIdentifierImpl.IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-
- List<IndexIdentifierImpl.IndexOperationMessage> ops;
-
- do{
- ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
- }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
-
- return ops;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
deleted file mode 100644
index ba0dc1f..0000000
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.queue;
-
-
-import org.junit.Assume;
-import org.junit.internal.runners.model.MultipleFailureException;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-import com.amazonaws.AmazonClientException;
-
-
-/**
- * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
- * https://github.com/junit-team/junit/issues/116
- *
- * Until then, simply marks as passed, which is a bit dangerous
- */
-public class NoAWSCredsRule implements TestRule {
-
- public Statement apply( final Statement base, final Description description ) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
-
- try {
- base.evaluate();
- }
- catch ( Throwable t ) {
-
- if ( !isMissingCredsException( t ) ) {
- throw t;
- }
-
- //do this so our test gets marked as ignored. Not pretty, but it works
- Assume.assumeTrue( false );
-
-
- }
- }
- };
- }
-
-
- private boolean isMissingCredsException( final Throwable t ) {
-
- if ( t instanceof AmazonClientException ) {
-
- final AmazonClientException ace = ( AmazonClientException ) t;
-
- if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
- "could not get aws secret key from system properties" ) ) {
- //swallow
- return true;
- }
- }
-
- /**
- * Handle the multiple failure junit trace
- */
- if( t instanceof MultipleFailureException ){
- for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
- final boolean isMissingCreds = isMissingCredsException( failure );
-
- if(isMissingCreds){
- return true;
- }
- }
- }
- final Throwable cause = t.getCause();
-
- if ( cause == null ) {
- return false;
- }
-
-
- return isMissingCredsException( cause );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index eecb9e1..452d328 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -27,16 +27,15 @@ import java.util.List;
import java.util.Map;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 6074e65..2f1ccdf 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -284,42 +284,42 @@
<artifactId>slf4j-api</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
<dependency>
<groupId>${project.parent.groupId}</groupId>
- <artifactId>queue</artifactId>
+ <artifactId>common</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-config</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-config</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index e6970b5..fbf8290 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -16,33 +16,28 @@
*/
package org.apache.usergrid.services.notifications;
-import org.apache.commons.io.IOUtils;
-import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
-import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
-import org.apache.usergrid.persistence.entities.Notifier;
+import java.io.InputStream;
+import java.lang.reflect.Field;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
-import org.apache.usergrid.services.notifications.ConnectionException;
-import org.apache.usergrid.services.notifications.NotificationsService;
-import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.net.SocketException;
+import org.apache.commons.io.IOUtils;
+
import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.entities.Notifier;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.services.AbstractServiceIT;
import org.apache.usergrid.services.ServiceAction;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertArrayEquals;
-
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class NotifiersServiceIT extends AbstractServiceIT {
[10/16] incubator-usergrid git commit: Removing references to
EntityManagerImpl to fix test runs.
Posted by sf...@apache.org.
Removing references to EntityManagerImpl to fix test runs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8dafe7b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8dafe7b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8dafe7b5
Branch: refs/heads/USERGRID-578
Commit: 8dafe7b579e4db934c1d612a9b185022df9c4e1c
Parents: 2ae6a3d
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 15:29:37 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 15:29:37 2015 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-core-context.xml | 20 +-------------------
1 file changed, 1 insertion(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8dafe7b5/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c5df523..b406b78 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -164,26 +164,8 @@
<constructor-arg ref="properties"/>
</bean>
-
-
- <bean id="entityManager" class="org.apache.usergrid.persistence.cassandra.EntityManagerImpl" scope="prototype"/>
-
<bean id="mailUtils" class="org.apache.usergrid.utils.MailUtils" />
-
- <bean id="traceTagAspect" class="org.apache.usergrid.persistence.cassandra.util.TraceTagAspect"/>
-
- <aop:config>
- <aop:aspect id="traceParticipant" ref="traceTagAspect">
- <!-- pointcut on the prescense of the TraceParticipant annotation only -->
- <aop:pointcut id="emTraceParticipant"
- expression="execution(* org.apache.usergrid.persistence.cassandra.*EntityManagerImpl.*(..)) and
- @annotation(org.apache.usergrid.persistence.cassandra.util.TraceParticipant)"/>
- <aop:around
- pointcut-ref="emTraceParticipant"
- method="applyTrace"/>
- </aop:aspect>
- </aop:config>
-
+
<!-- ============================================================== -->
<!-- Scheduler Settings from removed Scheduler Module's app context -->
<!-- ============================================================== -->
[11/16] incubator-usergrid git commit: Fixed ownership test along
with adding the graph mechanism to createIteminCollections!
Posted by sf...@apache.org.
Fixed ownership test along with adding the graph mechanism to createIteminCollections!
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4cc2e0bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4cc2e0bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4cc2e0bf
Branch: refs/heads/USERGRID-578
Commit: 4cc2e0bfb2cb55833a032d8725a345c1a425a7eb
Parents: 8dafe7b
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 16:21:26 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 16:21:26 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpRelationManager.java | 7 +++++--
.../org/apache/usergrid/persistence/EntityManagerIT.java | 8 ++------
2 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cc2e0bf/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index be8605e..a3d8172 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -505,8 +505,11 @@ public class CpRelationManager implements RelationManager {
addToCollection( collName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
- throw new UnsupportedOperationException( "Implement me directly in graph " );
-// getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
+ Id itemEntityId = new SimpleId( itemEntity.getUuid(),itemEntity.getType() );
+ final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collName, itemEntityId );
+
+ GraphManager gm = managerCache.getGraphManager( applicationScope );
+ gm.writeEdge( edge );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cc2e0bf/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index 3545739..f161040 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -26,19 +26,16 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.RandomStringUtils;
-
import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
@@ -46,7 +43,6 @@ import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
[07/16] incubator-usergrid git commit: Added factory for RX I/O
Scheduler.
Posted by sf...@apache.org.
Added factory for RX I/O Scheduler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9b9a1b02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9b9a1b02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9b9a1b02
Branch: refs/heads/USERGRID-578
Commit: 9b9a1b02f877ee6f35d7c3a7ba74001a17c237f1
Parents: 48d8060
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 08:01:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 10:36:06 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 5 +-
.../corepersistence/CpEntityManager.java | 9 +-
.../corepersistence/CpRelationManager.java | 57 +-------
.../index/AsyncIndexProvider.java | 15 +--
.../index/AsyncIndexService.java | 5 +-
.../index/InMemoryAsyncIndexService.java | 34 ++---
.../corepersistence/index/IndexServiceImpl.java | 9 +-
.../index/SQSAsyncIndexService.java | 4 +-
.../EntityCollectionManagerFactoryImpl.java | 15 ++-
.../impl/EntityCollectionManagerImpl.java | 38 +++---
.../mvcc/stage/write/RollbackAction.java | 2 -
.../persistence/core/guice/CommonModule.java | 12 ++
.../persistence/core/rx/RxSchedulerFig.java | 60 +++++++++
.../persistence/core/rx/RxTaskScheduler.java | 40 ++++++
.../core/rx/RxTaskSchedulerImpl.java | 129 +++++++++++++++++++
.../index/impl/IndexRefreshCommandImpl.java | 14 +-
.../persistence/index/impl/IndexingUtils.java | 6 +-
.../persistence/index/usergrid-mappings.json | 4 +-
18 files changed, 314 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 4758456..6ebff53 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -34,6 +34,9 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
+import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -144,7 +147,7 @@ public class CoreModule extends AbstractModule {
*****/
- bind(IndexService.class).to(IndexServiceImpl.class);
+ bind(IndexService.class).to( IndexServiceImpl.class );
//bind the queue provider
bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 844892a..6c3989d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -620,9 +620,8 @@ public class CpEntityManager implements EntityManager {
}
// update in all containing collections and connection indexes
- CpRelationManager rm = ( CpRelationManager ) getRelationManager( entity );
- rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
- timer.stop();
+
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity );
}
@@ -1067,9 +1066,7 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
- // update in all containing collections and connection indexes
- CpRelationManager rm = ( CpRelationManager ) getRelationManager( entityRef );
- rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index db9816e..be8605e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -255,47 +255,6 @@ public class CpRelationManager implements RelationManager {
}
- public void updateContainingCollectionAndCollectionIndexes(
- final org.apache.usergrid.persistence.model.entity.Entity cpEntity ) {
-
-
- throw new UnsupportedOperationException( "Use the new interface" );
-
-// final GraphManager gm = managerCache.getGraphManager( applicationScope );
-//
-// // loop through all types of edge to target
-//
-//
-// final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
-//
-// final EntityIndexBatch entityIndexBatch = ei.createBatch();
-//
-// final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
-//
-// // for each edge type, emit all the edges of that type
-// .flatMap( etype -> gm.loadEdgesToTarget(
-// new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
-// SearchByEdgeType.Order.DESCENDING, null ) ) )
-//
-// //for each edge we receive index and add to the batch
-// .doOnNext( edge -> {
-// // reindex the entity in the source entity's collection or connection index
-//
-// IndexEdge indexScope = generateScopeFromSource( edge );
-//
-// entityIndexBatch.index( indexScope, cpEntity );
-//
-// } ).doOnCompleted( () -> {
-// Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
-// entityIndexBatch.execute();
-// timeElasticIndexBatch.stop();
-// } ).count().toBlocking().lastOrDefault( 0 );
-//
-// //Adding graphite metrics
-//
-//
-// logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
- }
@Override
@@ -473,8 +432,6 @@ public class CpRelationManager implements RelationManager {
gm.writeEdge( edge ).toBlocking().last();
- //This is broken and needs fixed updateContainingCollectionAndCollectionIndexes See USERGRID-541
-
//perform indexing
@@ -482,7 +439,7 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- indexService.queueEntityIndexUpdate( applicationScope, memberEntity.getId(), memberEntity.getVersion() );
+ indexService.queueEntityIndexUpdate( applicationScope, memberEntity);
if ( logger.isDebugEnabled() ) {
@@ -490,17 +447,7 @@ public class CpRelationManager implements RelationManager {
itemRef.getUuid().toString(), itemRef.getType(), collName
} );
}
- // logger.debug("With head entity scope is {}:{}:{}", new Object[] {
- // headEntityScope.getApplication().toString(),
- // headEntityScope.getOwner().toString(),
- // headEntityScope.getName()});
-
- if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
- throw new UnsupportedOperationException( "Implement me directly in graph " );
-// getRelationManager( itemEntity )
-// .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
-// getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
- }
+
return itemEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 8257c94..77b4990 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.index;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
@@ -37,24 +37,23 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final QueryFig queryFig;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private final QueueManagerFactory queueManagerFactory;
private final MetricsFactory metricsFactory;
private final IndexService indexService;
+ private final RxTaskScheduler rxTaskScheduler;
private AsyncIndexService asyncIndexService;
@Inject
- public AsyncIndexProvider( final QueryFig queryFig,
- final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final QueueManagerFactory queueManagerFactory, final MetricsFactory metricsFactory,
- final IndexService indexService ) {
+ public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory, final
+ MetricsFactory metricsFactory,
+ final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
this.queryFig = queryFig;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -77,7 +76,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, entityCollectionManagerFactory );
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler );
case SQS:
return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
index d1f2fb6..06310ae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -40,8 +40,7 @@ public interface AsyncIndexService {
* We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
* After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
* @param applicationScope
- * @param entityId
- * @param version
+ * @param entity The entity to index
*/
- void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
index fdc9c65..3e2a271 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -20,56 +20,40 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
-import rx.schedulers.Schedulers;
@Singleton
public class InMemoryAsyncIndexService implements AsyncIndexService {
private final IndexService indexService;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final RxTaskScheduler rxTaskScheduler;
@Inject
- public InMemoryAsyncIndexService( final IndexService indexService,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {this.indexService = indexService;
+ public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ this.indexService = indexService;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
- final UUID version ) {
-
- final IndexEntityEvent event = new IndexEntityEvent( applicationScope, entityId, version );
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
//process the entity immediately
//only process the same version, otherwise ignore
- getEntity( applicationScope, entityId).filter( entity-> version.equals(entity.hasVersion() )).doOnNext( entity -> {
- indexService.indexEntity( applicationScope, entity );
- } ).subscribeOn( Schedulers.io() ).subscribe();
-
-
- }
-
- private Observable<Entity> getEntity( final ApplicationScope applicationScope, final Id entityId){
-
- final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- return ecm.load( entityId );
+ Observable.just( toIndex ).doOnNext( entity -> {
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 2a7533a..873e2b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -83,8 +83,8 @@ public class IndexServiceImpl implements IndexService {
//we always index in the target scope
final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
- //we may have to index
- final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeToTarget( edge ) );
+ //we may have to index we're indexing from source->target here
+ final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) );
//we might or might not need to index from target-> source
@@ -141,8 +141,9 @@ public class IndexServiceImpl implements IndexService {
/**
* An observable of sizes as we execute batches
+ *
+ * we're indexing from target->source here
*/
- return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
- .map( edge -> generateScopeFromSource( edge ) );
+ return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection ).map( edge -> generateScopeToTarget( edge ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
index cad20bd..dfcb97a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.QueueManager;
@@ -256,8 +257,7 @@ public class SQSAsyncIndexService implements AsyncIndexService {
@Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
- final UUID version ) {
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 9191c06..761c4b5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerial
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -74,6 +75,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
+ private final RxTaskScheduler rxTaskScheduler;
private LoadingCache<ApplicationScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
@@ -84,7 +86,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory,
+ rxTaskScheduler );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -95,8 +98,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
@Inject
- public EntityCollectionManagerFactoryImpl( final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
+ public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
final WriteOptimisticVerify writeOptimisticVerify,
final WriteCommit writeCommit, final RollbackAction rollback,
final MarkStart markStart, final MarkCommit markCommit,
@@ -105,9 +107,9 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- final EntityCacheFig entityCacheFig,
- MetricsFactory metricsFactory) {
+ @CollectionTaskExecutor final TaskExecutor taskExecutor, final
+ EntityCacheFig entityCacheFig,
+ MetricsFactory metricsFactory, final RxTaskScheduler rxTaskScheduler ) {
this.writeStart = writeStart;
this.writeVerifyUnique = writeVerifyUnique;
@@ -124,6 +126,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
this.metricsFactory = metricsFactory;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@Override
public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index f0b070c..6f10e86 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -49,6 +49,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSeria
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -109,6 +110,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
+ private final RxTaskScheduler rxTaskScheduler;
+
private final Keyspace keyspace;
private final Timer writeTimer;
private final Meter writeMeter;
@@ -125,26 +128,21 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Inject
- public EntityCollectionManagerImpl(
- final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit,
- final RollbackAction rollback,
- final MarkStart markStart,
- final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final ApplicationScope applicationScope,
- final MetricsFactory metricsFactory
-
- ) {
+ public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify, final WriteCommit
+ writeCommit,
+ final RollbackAction rollback, final MarkStart markStart,
+ final MarkCommit markCommit, final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final Keyspace keyspace, final EntityVersionTaskFactory entityVersionTaskFactory,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor, @Assisted final ApplicationScope applicationScope,
+ final MetricsFactory metricsFactory,
+
+ final RxTaskScheduler rxTaskScheduler ) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
+ this.rxTaskScheduler = rxTaskScheduler;
ValidationUtils.validateApplicationScope( applicationScope );
@@ -453,13 +451,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( writeVerifyUnique );
// optimistic verification
Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( writeOptimisticVerify );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
index cd15c26..8342e55 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
@@ -49,7 +49,6 @@ public class RollbackAction implements Action1<Throwable> {
private static final Logger log = LoggerFactory.getLogger( RollbackAction.class );
- private final Scheduler scheduler;
private final UniqueValueSerializationStrategy uniqueValueStrat;
private final MvccLogEntrySerializationStrategy logEntryStrat;
@@ -58,7 +57,6 @@ public class RollbackAction implements Action1<Throwable> {
public RollbackAction(MvccLogEntrySerializationStrategy logEntryStrat,
UniqueValueSerializationStrategy uniqueValueStrat ) {
- scheduler = Schedulers.io();
this.uniqueValueStrat = uniqueValueStrat;
this.logEntryStrat = logEntryStrat;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index bc84b6b..f266643 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@ -36,6 +36,9 @@ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerFig;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerImpl;
+import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import com.google.inject.AbstractModule;
import com.google.inject.Key;
@@ -86,6 +89,15 @@ public class CommonModule extends AbstractModule {
//do multibindings for migrations
//create the empty multibinder so other plugins can use it
Multibinder.newSetBinder( binder(), MigrationPlugin.class);
+
+
+ /**
+ * RX java scheduler configuration
+ */
+
+ install ( new GuicyFigModule( RxSchedulerFig.class ));
+
+ bind( RxTaskScheduler.class).to( RxTaskSchedulerImpl.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
new file mode 100644
index 0000000..7986132
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ *
+ */
+public interface RxSchedulerFig extends GuicyFig {
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String IO_SCHEDULER_THREADS = "scheduler.io.threads";
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String IO_SCHEDULER_NAME = "scheduler.io.poolName";
+
+
+
+
+ @Default( "100" )
+ @Key( IO_SCHEDULER_THREADS )
+ int getMaxIoThreads();
+
+ @Default( "Usergrid-RxIOPool" )
+ @Key(IO_SCHEDULER_NAME)
+ String getIoSchedulerName();
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
new file mode 100644
index 0000000..d6cc5e8
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import org.apache.usergrid.persistence.core.task.Task;
+
+import rx.Scheduler;
+
+
+/**
+ * An interface for returning task schedulers
+ */
+public interface RxTaskScheduler {
+
+ /**
+ * Get the scheduler for tasks that perform blocking I/O
+ * @return
+ */
+ Scheduler getAsyncIOScheduler();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
new file mode 100644
index 0000000..219cde6
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Scheduler;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * An implementation of the task scheduler that allows us to control the number of I/O threads
+ */
+@Singleton
+public class RxTaskSchedulerImpl implements RxTaskScheduler {
+
+ private static final Logger log = LoggerFactory.getLogger( RxTaskSchedulerImpl.class );
+
+ private final Scheduler scheduler;
+ private final String poolName;
+
+ @Inject
+ public RxTaskSchedulerImpl(final RxSchedulerFig schedulerFig){
+
+ this.poolName = schedulerFig.getIoSchedulerName();
+
+ final int poolSize = schedulerFig.getMaxIoThreads();
+
+
+ final BlockingQueue<Runnable> queue = new ArrayBlockingQueue<Runnable>(poolSize);
+
+
+ final MaxSizeThreadPool threadPool = new MaxSizeThreadPool( queue, poolSize );
+
+
+ this.scheduler = Schedulers.from(threadPool);
+
+
+ }
+
+
+ @Override
+ public Scheduler getAsyncIOScheduler() {
+ return scheduler;
+ }
+
+
+ /**
+ * Create a thread pool that will reject work if our audit tasks become overwhelmed
+ */
+ private final class MaxSizeThreadPool extends ThreadPoolExecutor {
+
+ public MaxSizeThreadPool( final BlockingQueue<Runnable> queue, final int maxPoolSize ) {
+
+ super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ),
+ new RejectedHandler() );
+ }
+ }
+
+
+ /**
+ * Thread factory that will name and count threads for easier debugging
+ */
+ private final class CountingThreadFactory implements ThreadFactory {
+
+ private final AtomicLong threadCounter = new AtomicLong();
+
+
+ @Override
+ public Thread newThread( final Runnable r ) {
+ final long newValue = threadCounter.incrementAndGet();
+
+ Thread t = new Thread( r, poolName + "-" + newValue );
+
+ t.setDaemon( true );
+
+ return t;
+ }
+ }
+
+
+ /**
+ * The handler that will handle rejected executions and signal the interface
+ */
+ private final class RejectedHandler implements RejectedExecutionHandler {
+
+
+ @Override
+ public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
+ log.warn( "{} task queue full, rejecting task {}", poolName, r );
+
+ //TODO T.N. do we want to run this on the caller thread?
+
+ throw new RejectedExecutionException( "Unable to run task, queue full" );
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index 07150bb..c997e5a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
@@ -48,7 +49,6 @@ import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import rx.Observable;
-import rx.schedulers.Schedulers;
import rx.util.async.Async;
@@ -64,12 +64,13 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer timer;
-
+ private final RxTaskScheduler rxTaskScheduler;
@Inject
public IndexRefreshCommandImpl( IndexIdentifier indexIdentifier, EsProvider esProvider,
IndexBufferConsumer producer, IndexFig indexFig, MetricsFactory metricsFactory,
- final IndexCache indexCache ) {
+ final IndexCache indexCache, final RxTaskScheduler rxTaskScheduler ) {
+
this.timer = metricsFactory.getTimer( IndexRefreshCommandImpl.class, "index.refresh.timer" );
@@ -78,6 +79,7 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
this.producer = producer;
this.indexFig = indexFig;
this.indexCache = indexCache;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -141,14 +143,14 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
logger.error( "Failed during refresh search for " + uuid, ee );
throw new RuntimeException( "Failed during refresh search for " + uuid, ee );
}
- }, Schedulers.io() ).call();
+ }, rxTaskScheduler.getAsyncIOScheduler() ).call();
return future.doOnNext( found -> {
if ( !found.hasFinished() ) {
- logger.error(String.format("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ logger.error("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime());
}else{
- logger.info(String.format("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ logger.info("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime());
}
} ).doOnCompleted(() -> {
//clean up our data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 7c33084..de5e29d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -54,11 +54,11 @@ public class IndexingUtils {
public static final String ENTITY_TYPE_FIELDNAME = "entityType";
- public static final String EDGE_NODE_ID_FIELDNAME = "edgeNodeId";
+ public static final String EDGE_NODE_ID_FIELDNAME = "nodeId";
public static final String EDGE_NAME_FIELDNAME = "edgeName";
- public static final String EDGE_NODE_TYPE_FIELDNAME = "edgeType";
+ public static final String EDGE_NODE_TYPE_FIELDNAME = "entityNodeType";
public static final String EDGE_TIMESTAMP_FIELDNAME = "edgeTimestamp";
@@ -107,8 +107,6 @@ public class IndexingUtils {
idString( sb, scope.getNodeId() );
sb.append( FIELD_SEPERATOR );
sb.append( scope.getEdgeName() );
- sb.append( FIELD_SEPERATOR );
- sb.append( scope.getNodeType() );
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index 4da1902..d4c34cd 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -21,7 +21,7 @@
"index": "not_analyzed",
"doc_values": true
},
- "edgeNodeId": {
+ "nodeId": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
@@ -31,7 +31,7 @@
"index": "not_analyzed",
"doc_values": true
},
- "edgeType": {
+ "entityNodeType": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
[05/16] incubator-usergrid git commit: Updated queue impl so that we
now send an async signal to index
Posted by sf...@apache.org.
Updated queue impl so that we now send an async signal to index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/745f3557
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/745f3557
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/745f3557
Branch: refs/heads/USERGRID-578
Commit: 745f3557843eff4a0006fa59913f7a4dbb592dff
Parents: 6f87fff
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 15:13:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 15:13:24 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 20 +-
.../corepersistence/CpEntityManager.java | 97 ++----
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../corepersistence/CpRelationManager.java | 8 +-
.../index/AsyncIndexProvider.java | 114 +++++++
.../index/AsyncIndexService.java | 47 +++
.../corepersistence/index/BufferQueue.java | 68 -----
.../index/BufferQueueInMemoryImpl.java | 116 -------
.../index/BufferQueueSQSImpl.java | 306 -------------------
.../index/InMemoryAsyncIndexService.java | 75 +++++
.../corepersistence/index/IndexEntityEvent.java | 75 +++++
.../index/IndexQueueService.java | 45 ---
.../corepersistence/index/QueueProvider.java | 112 -------
.../index/SQSAsyncIndexService.java | 263 ++++++++++++++++
.../index/SQSIndexQueueServiceImpl.java | 35 ---
.../index/BufferQueueSQSImplTest.java | 179 -----------
.../index/SQSAsyncIndexServiceTest.java | 179 +++++++++++
17 files changed, 802 insertions(+), 942 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7e8af87..4758456 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,20 +16,17 @@
package org.apache.usergrid.corepersistence;
-import org.apache.usergrid.corepersistence.index.BufferQueue;
-import org.apache.usergrid.corepersistence.index.IndexService;
-import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
-import org.apache.usergrid.corepersistence.index.QueryFig;
-import org.apache.usergrid.corepersistence.index.QueueProvider;
-import org.apache.usergrid.corepersistence.migration.*;
-import org.apache.usergrid.persistence.PersistenceModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.safehaus.guicyfig.GuicyFigModule;
-import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.corepersistence.index.AsyncIndexProvider;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.QueryFig;
+import org.apache.usergrid.corepersistence.migration.AppInfoMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
@@ -37,7 +34,6 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -47,12 +43,12 @@ import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
import com.google.inject.AbstractModule;
-import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -151,7 +147,7 @@ public class CoreModule extends AbstractModule {
bind(IndexService.class).to(IndexServiceImpl.class);
//bind the queue provider
- bind( BufferQueue.class).toProvider( QueueProvider.class );
+ bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
install( new GuicyFigModule( QueryFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1506551..844892a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -178,7 +179,7 @@ public class CpEntityManager implements EntityManager {
private final CounterUtils counterUtils;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -218,7 +219,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final IndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -722,7 +723,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Application app ) throws Exception {
- update(app);
+ update( app );
this.application = app;
}
@@ -903,7 +904,7 @@ public class CpEntityManager implements EntityManager {
}
- return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid()));
+ return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) );
}
@@ -970,7 +971,7 @@ public class CpEntityManager implements EntityManager {
public void setProperty(
EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
- setProperty(entityRef, propertyName, propertyValue, false);
+ setProperty( entityRef, propertyName, propertyValue, false );
}
@@ -988,7 +989,7 @@ public class CpEntityManager implements EntityManager {
entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
- entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID()));
+ entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
update( entity );
}
@@ -1083,7 +1084,7 @@ public class CpEntityManager implements EntityManager {
public void addToDictionary( EntityRef entityRef, String dictionaryName,
Object elementValue ) throws Exception {
- addToDictionary(entityRef, dictionaryName, elementValue, null);
+ addToDictionary( entityRef, dictionaryName, elementValue, null );
}
@@ -1144,7 +1145,7 @@ public class CpEntityManager implements EntityManager {
EntityRef entity = get( entityRef );
UUID timestampUuid = UUIDUtils.newTimeUUID();
- Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be);
+ Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(),
@@ -1241,7 +1242,7 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
HColumn<ByteBuffer, ByteBuffer> result =
@@ -1259,7 +1260,7 @@ public class CpEntityManager implements EntityManager {
}
}
else {
- logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null");
+ logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" );
}
return value;
@@ -1283,7 +1284,7 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
@@ -1340,7 +1341,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getDictionaries( EntityRef entity ) throws Exception {
- return getDictionaryNames(entity);
+ return getDictionaryNames( entity );
}
@@ -1386,7 +1387,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( get( entityId ))
- .getCollection(collectionName, query, resultsLevel);
+ .getCollection( collectionName, query, resultsLevel );
}
@@ -1496,7 +1497,7 @@ public class CpEntityManager implements EntityManager {
EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception {
return getRelationManager( connectingEntity )
- .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
+ .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
}
@@ -1785,14 +1786,14 @@ public class CpEntityManager implements EntityManager {
String roleTitle = batchRoleName;
String propertyName = groupId + ":" + batchRoleName;
Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
- properties.put("group", groupId);
+ properties.put( "group", groupId );
- Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties);
+ Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties );
getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) )
- .addToCollection(COLLECTION_ROLES, entity);
+ .addToCollection( COLLECTION_ROLES, entity );
- logger.info("Created role {} with id {} in group {}",
- new String[]{roleName, entity.getUuid().toString(), groupId.toString()});
+ logger.info( "Created role {} with id {} in group {}",
+ new String[] { roleName, entity.getUuid().toString(), groupId.toString() } );
return entity;
}
@@ -1883,7 +1884,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantUserPermission( UUID userId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission);
+ addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission );
}
@@ -1907,9 +1908,9 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
EntityRef userRef = userRef( userId );
EntityRef roleRef = getRoleRef( roleName );
- addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName);
- addToCollection(userRef, COLLECTION_ROLES, roleRef);
- addToCollection(roleRef, COLLECTION_USERS, userRef);
+ addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName );
+ addToCollection( userRef, COLLECTION_ROLES, roleRef );
+ addToCollection( roleRef, COLLECTION_USERS, userRef );
}
@@ -1945,7 +1946,7 @@ public class CpEntityManager implements EntityManager {
private EntityRef getRoleRef( String roleName ) throws Exception {
Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ));
+ Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2191,8 +2192,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getCounterNames() throws Exception {
Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
- Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS));
- names.addAll(nameSet);
+ Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) );
+ names.addAll( nameSet );
return names;
}
@@ -2264,7 +2265,7 @@ public class CpEntityManager implements EntityManager {
public Entity get( UUID uuid ) throws Exception {
MapManager mm = getMapManagerForTypes();
- String entityType = mm.getString(uuid.toString());
+ String entityType = mm.getString( uuid.toString() );
final Entity entity;
@@ -2288,7 +2289,7 @@ public class CpEntityManager implements EntityManager {
final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
- MapManager mm = managerCache.getMapManager(ms);
+ MapManager mm = managerCache.getMapManager( ms );
return mm;
}
@@ -2391,7 +2392,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
}
@@ -2406,8 +2407,8 @@ public class CpEntityManager implements EntityManager {
@Override
public void addGroupToRole( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName);
- addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName));
+ addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName );
+ addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) );
}
@@ -2428,7 +2429,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantGroupPermission( UUID groupId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission);
+ addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission );
}
@@ -2891,38 +2892,6 @@ public class CpEntityManager implements EntityManager {
} );
}
-
- void indexEntityIntoCollection( final Edge edge, final org.apache.usergrid.persistence.model.entity.Entity collectionMember) {
-
- throw new UnsupportedOperationException( "Use the new interface!" );
-//
-// final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
-// final EntityIndexBatch batch = aie.createBatch();
-//
-// // index member into entity collection | type scope
-// final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
-// batch.index( collectionIndexScope, collectionMember );
-//
-// //TODO REMOVE INDEX CODE
-// // // index member into entity | all-types scope
-// // IndexScope entityAllTypesScope = new IndexScopeImpl(
-// // collectionEntity.getId(),
-// // CpNamingUtils.ALL_TYPES, entityType );
-// //
-// // batch.index(entityAllTypesScope, memberEntity);
-// //
-// // // index member into application | all-types scope
-// // IndexScope appAllTypesScope = new IndexScopeImpl(
-// // getApplicationScope().getApplication(),
-// // CpNamingUtils.ALL_TYPES, entityType );
-// //
-// // batch.index(appAllTypesScope, memberEntity);
-//
-// //Adding graphite metrics
-// Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
-// batch.execute();
-// timeIndexEntityCollection.stop();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6ad8616..da1febc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -40,6 +40,7 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
@@ -111,7 +112,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector) {
@@ -123,7 +124,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.indexService = injector.getInstance( IndexService.class );
+ this.indexService = injector.getInstance( AsyncIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index a0a44b3..db9816e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
@@ -120,13 +121,13 @@ public class CpRelationManager implements RelationManager {
private final ApplicationScope applicationScope;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final IndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
Assert.notNull( em, "Entity manager cannot be null" );
@@ -481,7 +482,8 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- ( ( CpEntityManager ) em ).indexEntityIntoCollection( edge, memberEntity );
+ indexService.queueEntityIndexUpdate( applicationScope, memberEntity.getId(), memberEntity.getVersion() );
+
if ( logger.isDebugEnabled() ) {
logger.debug( "Added entity {}:{} to collection {}", new Object[] {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
new file mode 100644
index 0000000..8257c94
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class AsyncIndexProvider implements Provider<AsyncIndexService> {
+
+ private final QueryFig queryFig;
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final QueueManagerFactory queueManagerFactory;
+ private final MetricsFactory metricsFactory;
+ private final IndexService indexService;
+
+ private AsyncIndexService asyncIndexService;
+
+
+ @Inject
+ public AsyncIndexProvider( final QueryFig queryFig,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final QueueManagerFactory queueManagerFactory, final MetricsFactory metricsFactory,
+ final IndexService indexService ) {
+ this.queryFig = queryFig;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.queueManagerFactory = queueManagerFactory;
+ this.metricsFactory = metricsFactory;
+ this.indexService = indexService;
+ }
+
+
+ @Override
+ @Singleton
+ public AsyncIndexService get() {
+ if ( asyncIndexService == null ) {
+ asyncIndexService = getIndexService();
+ }
+
+
+ return asyncIndexService;
+ }
+
+
+ private AsyncIndexService getIndexService() {
+ final String value = queryFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new InMemoryAsyncIndexService( indexService, entityCollectionManagerFactory );
+ case SQS:
+ return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
new file mode 100644
index 0000000..d1f2fb6
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface AsyncIndexService {
+
+
+ /**
+ * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
+ * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
+ * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
+ * @param applicationScope
+ * @param entityId
+ * @param version
+ */
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
deleted file mode 100644
index fc1bdb7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-
-
-/**
- * A temporary interface of our buffer Q to decouple of producer and consumer;
- */
-public interface BufferQueue {
-
- /**
- * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
- * Other queues may return the future after ack on the offer. See the implementation documentation for details.
- * @param operation
- */
- public void offer(final IndexOperationMessage operation);
-
-
- /**
- * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
- * May return less than the take size, but will never return null
- *
- * @param takeSize
- * @param timeout
- * @param timeUnit
- * @return A null safe lid
- */
- public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
-
-
- /**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
- * This will set the future as done in in memory operations
- *
- * @param messages
- */
- public void ack(final List<IndexOperationMessage> messages);
-
- /**
- * Mark these message as failed. Set the exception in the future on local operation
- *
- * @param messages
- */
- public void fail(final List<IndexOperationMessage> messages, final Throwable t);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
deleted file mode 100644
index 0e43da3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class BufferQueueInMemoryImpl implements BufferQueue {
-
-
- private final QueryFig fig;
- private final ArrayBlockingQueue<IndexOperationMessage> messages;
-
-
- @Inject
- public BufferQueueInMemoryImpl( final QueryFig fig ) {
- this.fig = fig;
- messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
- }
-
-
- @Override
- public void offer( final IndexOperationMessage operation ) {
- try {
- messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
- }
- catch ( InterruptedException e ) {
- throw new RuntimeException("Unable to offer message to queue", e);
- }
- }
-
-
- @Override
- public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
- try {
-
-
- messages.drainTo( response, takeSize );
-
- //we got something, go process it
- if ( response.size() > 0 ) {
- return response;
- }
-
-
- final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
-
- if ( polled != null ) {
- response.add( polled );
-
- //try to add more
- messages.drainTo( response, takeSize - 1 );
- }
- }
- catch ( InterruptedException e ) {
- //swallow
- }
-
-
- return response;
- }
-
-
- @Override
- public void ack( final List<IndexOperationMessage> messages ) {
- //if we have a future ack it
- for ( final IndexOperationMessage op : messages ) {
- op.done();
- }
- }
-
-
- @Override
- public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
-
-
- for ( final IndexOperationMessage op : messages ) {
- final BetterFuture<IndexOperationMessage> future = op.getFuture();
-
- if ( future != null ) {
- future.setError( t );
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
deleted file mode 100644
index d955014..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
- * performing
- */
-@Singleton
-public class BufferQueueSQSImpl implements BufferQueue {
-
- private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final MapManager mapManager;
- private final QueryFig queryFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- final QueueScope queueScope =
- new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.queryFig = queryFig;
-
- final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
-
- this.mapManager = mapManagerFactory.createMapManager( scope );
-
-
- this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- @Override
- public void offer( final IndexOperationMessage operation ) {
-
- //no op
- if(operation.isEmpty()){
- operation.getFuture().done();
- return;
- }
-
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //write to cassandra
- this.mapManager.putString( identifier.toString(), payLoad, TTL );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- operation.done();
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
-
- final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if(messages.size() == 0){
- return response;
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
- //look up the values
- final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String key = getMessageKey( message );
-
- //now see if the key was there
- final String payload = storedCommands.get( key );
-
- //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
- // a DLQ
-
- if ( payload == null ) {
- continue;
- }
-
- final IndexOperationMessage messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public void ack( final List<IndexOperationMessage> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexOperationMessage ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
-
- //remove it from the map
- mapManager.delete( key );
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- @Override
- public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
- //no op, just let it retry after the queue timeout
- }
-
-
- /** Read the object from Base64 string. */
- private IndexOperationMessage fromString( String s ) throws IOException {
- IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexOperationMessage o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
- private String getMessageKey(final QueueMessage message){
- return message.getBody().toString();
- }
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexOperationMessage {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexOperationMessage source ) {
- this.message = message;
- this.addAllDeIndexRequest( source.getDeIndexRequests() );
- this.addAllIndexRequest( source.getIndexRequests() );
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
new file mode 100644
index 0000000..fdc9c65
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+
+@Singleton
+public class InMemoryAsyncIndexService implements AsyncIndexService {
+
+ private final IndexService indexService;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public InMemoryAsyncIndexService( final IndexService indexService,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {this.indexService = indexService;
+
+
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
+ final UUID version ) {
+
+ final IndexEntityEvent event = new IndexEntityEvent( applicationScope, entityId, version );
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+
+ getEntity( applicationScope, entityId).filter( entity-> version.equals(entity.hasVersion() )).doOnNext( entity -> {
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ }
+
+ private Observable<Entity> getEntity( final ApplicationScope applicationScope, final Id entityId){
+
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+ return ecm.load( entityId );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
new file mode 100644
index 0000000..9f81f34
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.Serializable;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * The immutable serializable event that represents and index operations
+ */
+public class IndexEntityEvent implements Serializable {
+
+ public ApplicationScope applicationScope;
+ public Id entityId;
+ public UUID entityVersion;
+
+
+ public IndexEntityEvent( final ApplicationScope applicationScope, final Id entityId, final UUID entityVersion ) {
+ this.applicationScope = applicationScope;
+ this.entityId = entityId;
+ this.entityVersion = entityVersion;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+ public void setApplicationScope( final ApplicationScope applicationScope ) {
+ this.applicationScope = applicationScope;
+ }
+
+
+ public Id getEntityId() {
+ return entityId;
+ }
+
+
+ public void setEntityId( final Id entityId ) {
+ this.entityId = entityId;
+ }
+
+
+ public UUID getEntityVersion() {
+ return entityVersion;
+ }
+
+
+ public void setEntityVersion( final UUID entityVersion ) {
+ this.entityVersion = entityVersion;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
deleted file mode 100644
index 8d7f222..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Low level queue service for indexing entities
- */
-public interface IndexQueueService {
-
-
- /**
- * Queue an entity to be index asynchronously
- * @param applicationScope
- * @param entityId
- * @param version
- */
- void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
deleted file mode 100644
index d3920db..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-
-
-/**
- * A provider to allow users to configure their queue impl via properties
- */
-@Singleton
-public class QueueProvider implements Provider<BufferQueue> {
-
- private final QueryFig queryFig;
-
- private final QueueManagerFactory queueManagerFactory;
- private final MapManagerFactory mapManagerFactory;
- private final MetricsFactory metricsFactory;
-
- private BufferQueue bufferQueue;
-
-
- @Inject
- public QueueProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- this.queryFig = queryFig;
-
-
- this.queueManagerFactory = queueManagerFactory;
- this.mapManagerFactory = mapManagerFactory;
- this.metricsFactory = metricsFactory;
- }
-
-
- @Override
- @Singleton
- public BufferQueue get() {
- if ( bufferQueue == null ) {
- bufferQueue = getQueue();
- }
-
-
- return bufferQueue;
- }
-
-
- private BufferQueue getQueue() {
- final String value = queryFig.getQueueImplementation();
-
- final Implementations impl = Implementations.valueOf( value );
-
- switch ( impl ) {
- case LOCAL:
- return new BufferQueueInMemoryImpl( queryFig );
- case SQS:
- return new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
- default:
- throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
- }
- }
-
-
- private String getErrorValues() {
- String values = "";
-
- for ( final Implementations impl : Implementations.values() ) {
- values += impl + ", ";
- }
-
- values = values.substring( 0, values.length() - 2 );
-
- return values;
- }
-
-
- /**
- * Different implementations
- */
- public static enum Implementations {
- LOCAL,
- SQS;
-
-
- public String asString() {
- return toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
new file mode 100644
index 0000000..cad20bd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class SQSAsyncIndexService implements AsyncIndexService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( SQSAsyncIndexService.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public SQSAsyncIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ this.writeTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ public void offer( final IndexEntityEvent operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+ final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if ( messages.size() == 0 ) {
+ return Collections.emptyList();
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String payload = getBody( message );
+
+ //now see if the key was there
+
+
+ final IndexEntityEvent messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public void ack( final List<IndexEntityEvent> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexEntityEvent ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexEntityEvent fromString( String s ) throws IOException {
+ IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexEntityEvent o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+
+ private String getBody( final QueueMessage message ) {
+ return message.getBody().toString();
+ }
+
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexEntityEvent {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
+ super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
+ this.message = message;
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
+ final UUID version ) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
deleted file mode 100644
index 42f36b1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class SQSIndexQueueServiceImpl implements IndexQueueService {
-
- @Override
- public void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version ) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
deleted file mode 100644
index a76a589..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.corepersistence.TestIndexModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
-import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.index.impl.IndexRequest;
-import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-
-import com.google.inject.Inject;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
-@NotThreadSafe
-public class BufferQueueSQSImplTest {
-
-
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
-
-
- @Rule
- public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
-
- @Inject
- public QueueManagerFactory queueManagerFactory;
-
- @Inject
- public QueryFig queryFig;
-
- @Inject
- public MapManagerFactory mapManagerFactory;
-
- @Inject
- public MetricsFactory metricsFactory;
-
-
- private BufferQueueSQSImpl bufferQueueSQS;
-
- @Before
- public void setup(){
- bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
- }
-
-
-
-
- @Test
- public void testMessageIndexing(){
-
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
- final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
- assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
-
- final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
-
-
- final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
-
-
- //de-index request
- final DeIndexRequest
- deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
-
-
- SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
-
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
- SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
-
-
-
-
- IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
- indexOperationMessage.addIndexRequest( indexRequest1);
- indexOperationMessage.addIndexRequest( indexRequest2);
-
- indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
- indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
-
- bufferQueueSQS.offer( indexOperationMessage );
-
- //wait for it to send to SQS
- indexOperationMessage.getFuture().get();
-
- //now get it back
-
- final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
-
- assertTrue(ops.size() > 0);
-
- final IndexOperationMessage returnedOperation = ops.get( 0 );
-
- //get the operations out
-
- final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
-
- assertTrue(indexRequestSet.contains(indexRequest1));
- assertTrue(indexRequestSet.contains(indexRequest2));
-
-
- final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
-
- assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
- assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
-
-
-
- //now ack the message
-
- bufferQueueSQS.ack( ops );
-
- }
-
- private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-
- List<IndexOperationMessage> ops;
-
- do{
- ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
- }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
-
- return ops;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
new file mode 100644
index 0000000..0977caa
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+@NotThreadSafe
+public class SQSAsyncIndexServiceTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+ @Inject
+ public QueueManagerFactory queueManagerFactory;
+
+ @Inject
+ public QueryFig queryFig;
+
+
+ @Inject
+ public MetricsFactory metricsFactory;
+
+
+ private SQSAsyncIndexService bufferQueueSQS;
+
+ @Before
+ public void setup(){
+ bufferQueueSQS = new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ }
+
+
+
+
+ @Test
+ public void testMessageIndexing(){
+
+ fail("fix me");
+// ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+// final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+// assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
+// assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
+//
+// final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
+// final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
+//
+//
+// final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
+// final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
+//
+//
+// //de-index request
+// final DeIndexRequest
+// deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
+//
+//
+// SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
+//
+// final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
+// SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
+//
+//
+//
+//
+// IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
+// indexOperationMessage.addIndexRequest( indexRequest1);
+// indexOperationMessage.addIndexRequest( indexRequest2);
+//
+// indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
+// indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
+//
+// bufferQueueSQS.offer( indexOperationMessage );
+//
+// //wait for it to send to SQS
+// indexOperationMessage.getFuture().get();
+//
+// //now get it back
+//
+// final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
+//
+// assertTrue(ops.size() > 0);
+//
+// final IndexOperationMessage returnedOperation = ops.get( 0 );
+//
+// //get the operations out
+//
+// final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
+//
+// assertTrue(indexRequestSet.contains(indexRequest1));
+// assertTrue(indexRequestSet.contains(indexRequest2));
+//
+//
+// final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
+//
+// assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
+// assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
+//
+//
+//
+// //now ack the message
+//
+// bufferQueueSQS.ack( ops );
+
+ }
+
+// private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
+// final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+//
+// List<IndexOperationMessage> ops;
+//
+// do{
+// ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+// }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
+//
+// return ops;
+// }
+
+
+
+
+}
[09/16] incubator-usergrid git commit: Added equivalent two-dot-o
search/write timeouts to ES Search and Buffer Consumer.
Posted by sf...@apache.org.
Added equivalent two-dot-o search/write timeouts to ES Search and Buffer Consumer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2ae6a3d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2ae6a3d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2ae6a3d1
Branch: refs/heads/USERGRID-578
Commit: 2ae6a3d1562e999af94c9f13dd5db81558d6ef0b
Parents: 078666e
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 10:06:33 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 10:06:33 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexFig.java | 15 +++++++++++++++
.../index/impl/EsApplicationEntityIndexImpl.java | 4 +++-
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index d13d055..47b25ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -71,6 +71,13 @@ public interface IndexFig extends GuicyFig {
String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ /**
+ * Timeout calls to elasticsearch.
+ * @return
+ */
+ public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+
+ public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
/**
* The client type to use. Valid values are NODE or TRANSPORT
@@ -177,5 +184,13 @@ public interface IndexFig extends GuicyFig {
@Key("elasticsearch.refresh_sleep_ms")
@Default("200")
long refreshSleep();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_QUERY_TIMEOUT )
+ long getQueryTimeout();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_WRITE_TIMEOUT )
+ long getWriteTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 0986ff6..43b55d0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -95,6 +95,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final SearchRequestBuilderStrategy searchRequest;
private FailureMonitor failureMonitor;
private final int cursorTimeout;
+ private final long queryTimeout;
@Inject
@@ -118,6 +119,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
this.searchTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.cursor.timer" );
this.cursorTimeout = config.getQueryCursorTimeout();
+ this.queryTimeout = config.getWriteTimeout();
this.deleteApplicationTimer =
metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "delete.application" );
@@ -163,7 +165,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
+ searchResponse = srb.execute().actionGet(queryTimeout);
timeSearch.stop();
}
catch ( Throwable t ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 6bcc405..7cc7dcf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -209,7 +209,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
try {
- responses = bulkRequest.execute().actionGet();
+ responses = bulkRequest.execute().actionGet(indexFig.getWriteTimeout());
}
catch ( Throwable t ) {
log.error( "Unable to communicate with elasticsearch" );
[15/16] incubator-usergrid git commit: Start of rewire of the rebuild
process
Posted by sf...@apache.org.
Start of rewire of the rebuild process
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3480a363
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3480a363
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3480a363
Branch: refs/heads/USERGRID-578
Commit: 3480a3637cc3cc6c79504642c3c44fbcf5878efd
Parents: ee1a66f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 19:52:51 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 19:52:51 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 +-
.../corepersistence/CpEntityManager.java | 85 +-----
.../corepersistence/CpEntityManagerFactory.java | 119 +++-----
.../corepersistence/CpRelationManager.java | 9 +-
.../index/AsyncIndexProvider.java | 12 +-
.../index/AsyncIndexService.java | 49 ----
.../index/AsyncReIndexService.java | 42 +++
.../index/InMemoryAsyncIndexService.java | 92 -------
.../index/InMemoryAsyncReIndexService.java | 89 ++++++
.../corepersistence/index/ReIndexAction.java | 33 +++
.../corepersistence/index/ReIndexService.java | 39 ++-
.../index/ReIndexServiceImpl.java | 43 ++-
.../index/SQSAsyncIndexService.java | 270 -------------------
.../index/SQSAsyncReIndexService.java | 269 ++++++++++++++++++
.../rx/impl/AllEntityIdsObservable.java | 5 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 4 +-
.../corepersistence/util/CpNamingUtils.java | 6 +-
.../usergrid/persistence/EntityManager.java | 5 -
.../persistence/EntityManagerFactory.java | 11 +-
.../index/SQSAsyncIndexServiceTest.java | 18 +-
.../PerformanceEntityRebuildIndexTest.java | 18 +-
.../cassandra/EntityManagerFactoryImplIT.java | 15 +-
.../graph/serialization/EdgesObservable.java | 5 +-
.../serialization/impl/EdgesObservableImpl.java | 6 +-
24 files changed, 584 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index bb936b5..51972d8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -22,7 +22,7 @@ import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.corepersistence.index.AsyncIndexProvider;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
import org.apache.usergrid.corepersistence.index.QueryFig;
@@ -37,9 +37,6 @@ import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
-import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
-import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -154,7 +151,7 @@ public class CoreModule extends AbstractModule {
bind( IndexService.class ).to( IndexServiceImpl.class );
//bind the queue provider
- bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
+ bind( AsyncReIndexService.class).toProvider( AsyncIndexProvider.class );
install( new GuicyFigModule( QueryFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6c3989d..72ca955 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,8 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -49,7 +48,6 @@ import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityFactory;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.Query;
@@ -68,7 +66,6 @@ import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -81,7 +78,6 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.map.MapManager;
@@ -179,7 +175,7 @@ public class CpEntityManager implements EntityManager {
private final CounterUtils counterUtils;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -219,7 +215,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncReIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -804,8 +800,8 @@ public class CpEntityManager implements EntityManager {
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
- Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields( Inflector.getInstance().singularize( collectionType ),
- Arrays.<Field>asList( uniqueLookupRepairField ) );
+ Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ), Arrays.<Field>asList( uniqueLookupRepairField ) );
if(fieldSetObservable == null){
logger.debug( "Couldn't return the observable based on unique entities." );
@@ -1736,7 +1732,7 @@ public class CpEntityManager implements EntityManager {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be);
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( roleName ), permission, timestamp );
+ getRolePermissionsKey( roleName ), permission, timestamp );
//Adding graphite metrics
Timer.Context timeRevokeRolePermission = entRevokeRolePermissionsTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -1747,8 +1743,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getRolePermissions( String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ),
- ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( roleName ) );
+ return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( roleName ) );
}
//TODO: does this need graphite monitoring
@@ -1830,8 +1826,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getGroupRolePermissions( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ),
- ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( groupId, roleName ) );
+ return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( groupId, roleName ) );
}
@@ -1840,7 +1836,7 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
removeFromDictionary( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ), DICTIONARY_ROLENAMES, roleName );
cass.deleteRow( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
- SimpleRoleRef.getIdForGroupIdAndRoleName( groupId, roleName ) );
+ SimpleRoleRef.getIdForGroupIdAndRoleName( groupId, roleName ) );
}
@@ -1931,7 +1927,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getGroupRoleRef( UUID groupId, String roleName ) throws Exception {
Results results = this.searchCollection( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
+ Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2830,64 +2826,7 @@ public class CpEntityManager implements EntityManager {
- /**
- * Completely reindex the named collection in the application associated with this EntityManager.
- */
- @Override
- public void reindexCollection(
- final EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
-
- CpWalker walker = new CpWalker( );
-
- walker.walkCollections( this, getApplication(), collectionName, reverse, new CpVisitor() {
-
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
-
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- // swallow this, it just means this was already updated, which accomplishes our task
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
- }
- } );
- }
-
-
- /**
- * Completely reindex the application associated with this EntityManager.
- */
- public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception {
-
- CpWalker walker = new CpWalker( );
-
- walker.walkCollections( this, getApplication(), null, false, new CpVisitor() {
-
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
- }
- } );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6d20b7..46db3f8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,24 +15,16 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.base.*;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
@@ -41,11 +33,19 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.AbstractEntity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@ -59,7 +59,6 @@ import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -67,17 +66,27 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
-import rx.Observable;
-import java.util.*;
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+import rx.Observable;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import static org.apache.usergrid.persistence.Schema.*;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
/**
@@ -113,7 +122,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector) {
@@ -125,7 +134,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.indexService = injector.getInstance( AsyncIndexService.class );
+ this.indexService = injector.getInstance( AsyncReIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
@@ -307,14 +316,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
migrateAppInfo(applicationId,CpNamingUtils.DELETED_APPLICATION_INFO,CpNamingUtils.APPLICATION_INFO)
.toBlocking().lastOrDefault(null);
- this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
- @Override
- public void onProgress(EntityRef entity) {
- logger.info("Restored entity {}:{}", entity.getType(), entity.getUuid());
- }
- });
+ throw new UnsupportedOperationException( "Implement index rebuild" );
- return managementEm.get(new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO,applicationId));
+// return managementEm.get(new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO,applicationId));
}
@Override
@@ -661,44 +665,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public void rebuildAllIndexes( ProgressObserver po ) throws Exception {
-
- logger.info("\n\nRebuilding all indexes\n");
-
- rebuildInternalIndexes( po );
-
- Map<String, UUID> appMap = getApplications();
-
- logger.info("About to rebuild indexes for {} applications", appMap.keySet().size());
-
- for ( UUID appUuid : appMap.values() ) {
- try {
- rebuildApplicationIndexes(appUuid, po);
- } catch ( Exception e) {
- logger.error("Error rebuilding index for app " + appUuid + " continuing...", e );
- }
- }
- }
-
-
- @Override
- public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
-
- // TODO: remove this after appinfo migration done
- rebuildApplicationIndexes( CpNamingUtils.SYSTEM_APP_ID, po);
- rebuildApplicationIndexes( CpNamingUtils.MANAGEMENT_APPLICATION_ID, po );
- }
-
-
- @Override
- public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
-
- EntityManager em = getEntityManager( appId );
- em.reindex( po );
-
- logger.info("\n\nRebuilt index for applicationId {} \n", appId);
- }
-
@Override
@@ -716,18 +682,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
- public void rebuildCollectionIndex(
- UUID appId, String collectionName, boolean reverse, ProgressObserver po ) throws Exception {
-
- EntityManager em = getEntityManager( appId );
-
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- Application app = em.getApplication();
-
- em.reindexCollection(po, collectionName, reverse);
-
- logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
- new Object[]{app.getName(), appId, collectionName});
+ public ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ) {
+
+ throw new UnsupportedOperationException( "Implement me" );
+//
+// EntityManager em = getEntityManager( appId );
+//
+// //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
+// Application app = em.getApplication();
+//
+// em.reindexCollection(po, collectionName, reverse);
+//
+// logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
+// new Object[]{app.getName(), appId, collectionName});
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index a3d8172..8f125ad 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,8 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
@@ -43,7 +42,6 @@ import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.RelationManager;
@@ -80,7 +78,6 @@ import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import rx.Observable;
-import rx.functions.Action1;
import rx.functions.Func1;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
@@ -121,13 +118,13 @@ public class CpRelationManager implements RelationManager {
private final ApplicationScope applicationScope;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncReIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
Assert.notNull( em, "Entity manager cannot be null" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index d00ef8e..2c48c13 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -35,7 +35,7 @@ import com.google.inject.Singleton;
* A provider to allow users to configure their queue impl via properties
*/
@Singleton
-public class AsyncIndexProvider implements Provider<AsyncIndexService> {
+public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
private final QueryFig queryFig;
@@ -46,7 +46,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final AllEntityIdsObservable allEntitiesObservable;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private AsyncIndexService asyncIndexService;
+ private AsyncReIndexService asyncIndexService;
@Inject
@@ -67,7 +67,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
@Override
@Singleton
- public AsyncIndexService get() {
+ public AsyncReIndexService get() {
if ( asyncIndexService == null ) {
asyncIndexService = getIndexService();
}
@@ -77,17 +77,17 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
}
- private AsyncIndexService getIndexService() {
+ private AsyncReIndexService getIndexService() {
final String value = queryFig.getQueueImplementation();
final Implementations impl = Implementations.valueOf( value );
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, rxTaskScheduler,
+ return new InMemoryAsyncReIndexService( indexService, rxTaskScheduler,
entityCollectionManagerFactory );
case SQS:
- return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ return new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
deleted file mode 100644
index 8b5ced1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * Low level queue service for indexing entities
- */
-public interface AsyncIndexService extends ReIndexService.IndexAction {
-
-
- /**
- * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
- * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
- * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
- * @param applicationScope
- * @param entity The entity to index
- */
- void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
new file mode 100644
index 0000000..c6eedd7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface AsyncReIndexService extends ReIndexAction {
+
+
+ /**
+ * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
+ * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
+ * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
+ * @param applicationScope
+ * @param entity The entity to index
+ */
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
deleted file mode 100644
index 0efb964..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-import rx.Observable;
-
-
-@Singleton
-public class InMemoryAsyncIndexService implements AsyncIndexService {
-
- private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncIndexService.class);
- private final IndexService indexService;
- private final RxTaskScheduler rxTaskScheduler;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
-
- @Inject
- public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.indexService = indexService;
- this.rxTaskScheduler = rxTaskScheduler;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
-
- //process the entity immediately
- //only process the same version, otherwise ignore
-
- Observable.just( toIndex ).doOnNext( entity -> {
- log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
- indexService.indexEntity( applicationScope, entity );
- } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
- }
-
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
-
- final Id entityId = entityIdScope.getId();
-
- final Entity
- entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
- entityId ).toBlocking().lastOrDefault( null );
-
-
- if(entity == null){
- log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
- }
-
- indexService.indexEntity(applicationScope, entity );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
new file mode 100644
index 0000000..5ebda87
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+
+
+@Singleton
+public class InMemoryAsyncReIndexService implements AsyncReIndexService {
+
+ private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncReIndexService.class);
+ private final IndexService indexService;
+ private final RxTaskScheduler rxTaskScheduler;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public InMemoryAsyncReIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.indexService = indexService;
+ this.rxTaskScheduler = rxTaskScheduler;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+
+ Observable.just( toIndex ).doOnNext( entity -> {
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
+ }
+
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+
+ final Id entityId = entityIdScope.getId();
+
+ final Entity
+ entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
+ entityId ).toBlocking().lastOrDefault( null );
+
+
+ if(entity == null){
+ log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
+ }
+
+ indexService.indexEntity( applicationScope, entity );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
new file mode 100644
index 0000000..086b2aa
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+
+
+/**
+ * Callback to perform an index operation based on an scope during bulk re-index operations
+ */
+@FunctionalInterface
+public interface ReIndexAction {
+
+ void index( final EntityIdScope entityIdScope );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index dca6cac..91409fe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,10 +20,12 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.FutureTask;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -31,6 +33,7 @@ import com.google.common.base.Optional;
import rx.Observable;
import rx.Observer;
+import rx.observables.ConnectableObservable;
/**
@@ -40,14 +43,14 @@ public interface ReIndexService {
/**
- * Reindex all applications using the cursor provided
- *
- * @param startTimestamp The timestamp to start seeking from
- *
- * @return a cursor that can be used to resume the operation on the next run
+ * Perform an index rebuild
+ * @param appId
+ * @param collection
+ * @return
*/
- IndexResponse reIndex( final rx.Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
- final Optional<Long> startTimestamp, final IndexAction indexAction );
+ IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> collectionName, final Optional<String> cursor,
+ final Optional<Long> startTimestamp );
+
/**
@@ -55,10 +58,10 @@ public interface ReIndexService {
*/
class IndexResponse {
final String cursor;
- final Observable<Long> indexedEdgecount;
+ final ConnectableObservable<EdgeScope> indexedEdgecount;
- public IndexResponse( final String cursor, final Observable<Long> indexedEdgecount ) {
+ public IndexResponse( final String cursor, final ConnectableObservable<EdgeScope> indexedEdgecount ) {
this.cursor = cursor;
this.indexedEdgecount = indexedEdgecount;
}
@@ -74,23 +77,13 @@ public interface ReIndexService {
/**
- * Return the observable long count of all edges indexed
+ * Return the observable of all edges to be indexed.
+ *
+ * Note that after subscribing "connect" will need to be called to ensure that processing begins
* @return
*/
- public Observable<Long> getCount() {
+ public ConnectableObservable<EdgeScope> getCount() {
return indexedEdgecount;
}
}
-
-
-
-
- /**
- * Callback to perform an index operation based on an scope during bulk re-index operations
- */
- @FunctionalInterface
- interface IndexAction {
-
- void index( final EntityIdScope entityIdScope );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 5c022e1..3553c87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -20,8 +20,10 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -37,11 +39,16 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
import rx.Observable;
import rx.observables.ConnectableObservable;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
+
+@Singleton
public class ReIndexServiceImpl implements ReIndexService {
private static final MapScope RESUME_MAP_SCOPTE =
@@ -51,48 +58,52 @@ public class ReIndexServiceImpl implements ReIndexService {
private static final int INDEX_TTL = 60 * 60 * 24 * 10;
+ private final AllApplicationsObservable allApplicationsObservable;
private final AllEntityIdsObservable allEntityIdsObservable;
private final QueryFig queryFig;
private final RxTaskScheduler rxTaskScheduler;
private final MapManager mapManager;
+ private final AsyncReIndexService indexService;
+ @Inject
public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
- final MapManagerFactory mapManagerFactory, final QueryFig queryFig,
- final RxTaskScheduler rxTaskScheduler ) {
+ final MapManagerFactory mapManagerFactory,
+ final AllApplicationsObservable allApplicationsObservable, final QueryFig queryFig,
+ final RxTaskScheduler rxTaskScheduler, final AsyncReIndexService indexService ) {
this.allEntityIdsObservable = allEntityIdsObservable;
+ this.allApplicationsObservable = allApplicationsObservable;
this.queryFig = queryFig;
this.rxTaskScheduler = rxTaskScheduler;
+ this.indexService = indexService;
this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPTE );
}
- @Override
- public IndexResponse reIndex( final Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
- final Optional<Long> startTimestamp, final IndexAction indexAction ) {
+ @Override
+ public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection,
+ final Optional<String> collectionName, final Optional<String> cursor,
+ final Optional<Long> startTimestamp ) {
//load our last emitted Scope if a cursor is present
if ( cursor.isPresent() ) {
throw new UnsupportedOperationException( "Build this" );
}
+
+ final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( getApplicationScope(appId.get()) ) : allApplicationsObservable.getData();
+
final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
//create an observable that loads each entity and indexes it, start it running with publish
final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, startTimestamp )
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, collectionName, startTimestamp )
//for each edge, create our scope and index on it
- .doOnNext( edge -> indexAction
- .index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) )
+ .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
- .subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).publish();
-
-
- //count our longs
- final Observable<Long> indexedCount = runningReIndex.countLong();
//start our sampler and state persistence
@@ -107,7 +118,11 @@ public class ReIndexServiceImpl implements ReIndexService {
} ).subscribe();
- return new IndexResponse( newCursor, indexedCount );
+ //start pushing to both
+ runningReIndex.connect();
+
+
+ return new IndexResponse( newCursor, runningReIndex );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
deleted file mode 100644
index 6d06637..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class SQSAsyncIndexService implements AsyncIndexService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( SQSAsyncIndexService.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final QueryFig queryFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public SQSAsyncIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
- final MetricsFactory metricsFactory ) {
- final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.queryFig = queryFig;
-
- this.writeTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- public void offer( final IndexEntityEvent operation ) {
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
- final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if ( messages.size() == 0 ) {
- return Collections.emptyList();
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String payload = getBody( message );
-
- //now see if the key was there
-
-
- final IndexEntityEvent messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- public void ack( final List<IndexEntityEvent> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexEntityEvent ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- /** Read the object from Base64 string. */
- private IndexEntityEvent fromString( String s ) throws IOException {
- IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexEntityEvent o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
-
- private String getBody( final QueueMessage message ) {
- return message.getBody().toString();
- }
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- }
-
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexEntityEvent {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
- super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
- this.message = message;
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
- throw new UnsupportedOperationException( "Implement me" );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
new file mode 100644
index 0000000..60a804c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class SQSAsyncReIndexService implements AsyncReIndexService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( SQSAsyncReIndexService.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public SQSAsyncReIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ this.writeTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ public void offer( final IndexEntityEvent operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+ final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if ( messages.size() == 0 ) {
+ return Collections.emptyList();
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String payload = getBody( message );
+
+ //now see if the key was there
+
+
+ final IndexEntityEvent messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public void ack( final List<IndexEntityEvent> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexEntityEvent ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexEntityEvent fromString( String s ) throws IOException {
+ IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexEntityEvent o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+
+ private String getBody( final QueueMessage message ) {
+ return message.getBody().toString();
+ }
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ }
+
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexEntityEvent {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
+ super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
+ this.message = message;
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
+ throw new UnsupportedOperationException( "Implement index rebuild" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
index c805a59..b9e5373 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -43,9 +43,10 @@ public interface AllEntityIdsObservable {
/**
* Get all edges that represent edges to entities in the system
* @param appScopes
- * @param startTime The time to
+ * @param edgeType The edge type to use (if specified)
+ * @param startTime The time to start with
* @return
*/
- Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<Long> startTime);
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index f9df1f5..257fab1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -81,12 +81,12 @@ public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
@Override
- public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<Long> startTime) {
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime) {
return appScopes.flatMap( applicationScope -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(), startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
+ return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(),edgeType, startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 67cc0ca..c42ad10 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -205,9 +205,7 @@ public class CpNamingUtils {
public static ApplicationScope getApplicationScope( UUID applicationId ) {
// We can always generate a scope, it doesn't matter if the application exists yet or not.
- final ApplicationScopeImpl scope = new ApplicationScopeImpl( generateApplicationId( applicationId ) );
-
- return scope;
+ return new ApplicationScopeImpl( generateApplicationId( applicationId ) );
}
@@ -229,6 +227,8 @@ public class CpNamingUtils {
return generateApplicationId( MANAGEMENT_APPLICATION_ID );
}
+
+
/**
* Get the map scope for the applicationId to store entity uuid to type mapping
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index fc8b3d5..23b6d6b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -692,11 +692,6 @@ public interface EntityManager {
/** For testing purposes */
public void flushManagerCaches();
- void reindexCollection(
- EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception;
-
- public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
-
public Entity getUniqueEntityFromAlias( String aliasType, String aliasValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index e70cd0d..b3f4b62 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -21,6 +21,8 @@ import java.util.Map;
import java.util.UUID;
import com.google.common.base.Optional;
+
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import rx.Observable;
@@ -163,12 +165,6 @@ public interface EntityManagerFactory {
public IndexRefreshCommand.IndexRefreshCommandInfo refreshIndex();
- public void rebuildAllIndexes( ProgressObserver po ) throws Exception;
-
- public void rebuildInternalIndexes( ProgressObserver po ) throws Exception;
-
- public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception;
-
/**
* Perform a realtime count of every entity in the system. This can be slow as it traverses the entire system graph
@@ -178,8 +174,7 @@ public interface EntityManagerFactory {
/** For testing purposes */
public void flushEntityManagerCaches();
- void rebuildCollectionIndex(
- UUID appId, String collection, boolean reverse, ProgressObserver po) throws Exception;
+ ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection );
/**
* Add a new index to the application for scale
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
index 0977caa..5e9a5a1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
@@ -20,19 +20,9 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
import org.apache.usergrid.corepersistence.TestIndexModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.index.impl.IndexRequest;
-import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
+
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -41,10 +31,8 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
@@ -80,11 +68,11 @@ public class SQSAsyncIndexServiceTest {
public MetricsFactory metricsFactory;
- private SQSAsyncIndexService bufferQueueSQS;
+ private SQSAsyncReIndexService bufferQueueSQS;
@Before
public void setup(){
- bufferQueueSQS = new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ bufferQueueSQS = new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index b0fc245..a293a0c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -213,11 +213,12 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- // do it forwards
- setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
-
- // and backwards, just to make sure both cases are covered
- setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", true, po );
+ fail( "Implement index rebuild" );
+// // do it forwards
+// setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
+//
+// // and backwards, just to make sure both cases are covered
+// setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", true, po );
reporter.report();
registry.remove( meterName );
@@ -354,9 +355,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- setup.getEmf().rebuildInternalIndexes( po );
-
- setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+ fail( "Implement index rebuild" );
+// setup.getEmf().rebuildInternalIndexes( po );
+//
+// setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
reporter.report();
registry.remove( meterName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index 1a0a13b..2452606 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -52,6 +52,8 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
@NotThreadSafe
public class EntityManagerFactoryImplIT extends AbstractCoreIT {
@@ -175,12 +177,13 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
// restore the app
emf.restoreApplication(deletedAppId);
- emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
- @Override
- public void onProgress(EntityRef entity) {
- logger.debug("Reindexing {}:{}", entity.getType(), entity.getUuid());
- }
- });
+ fail( "Implement index rebuild" );
+// emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
+// @Override
+// public void onProgress(EntityRef entity) {
+// logger.debug("Reindexing {}:{}", entity.getType(), entity.getUuid());
+// }
+// });
this.app.refreshIndex();
// test to see that app now works and is happy
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 9e7b8e6..9f0bd60 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -45,10 +45,11 @@ public interface EdgesObservable {
* Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
* @param gm
* @param sourceNode
- * @param startTimestamp
+ * @param edgeType The edge type if specified. Otherwise all types will be used
+ * @param startTimestamp The start timestamp if specfiied, otherwise Long.MIN will be used
* @return
*/
- Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,final Optional<String> edgeType,
final Optional<Long> startTimestamp );
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 859ca2e..ca9fb03 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -71,10 +71,12 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
- public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode, final Optional<String> edgeTypeInput,
final Optional<Long> startTimestamp ) {
- final Observable<String> edgeTypes =
+
+
+ final Observable<String> edgeTypes = edgeTypeInput.isPresent()? Observable.just( edgeTypeInput.get() ):
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
[12/16] incubator-usergrid git commit: Refactored observables
Posted by sf...@apache.org.
Refactored observables
Refactored re-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/59455748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/59455748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/59455748
Branch: refs/heads/USERGRID-578
Commit: 59455748c0978212396b4bf8f53bc410af81f922
Parents: 078666e
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 18:27:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 18:27:19 2015 -0600
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 35 ------
.../usergrid/corepersistence/CoreModule.java | 1 +
.../events/EntityVersionDeletedHandler.java | 4 +-
.../index/AsyncIndexProvider.java | 17 ++-
.../index/AsyncIndexService.java | 5 +-
.../index/InMemoryAsyncIndexService.java | 39 ++++++-
.../corepersistence/index/QueryFig.java | 5 +
.../corepersistence/index/ReIndexService.java | 96 ++++++++++++++++
.../index/ReIndexServiceImpl.java | 114 +++++++++++++++++++
.../index/SQSAsyncIndexService.java | 9 +-
.../rx/impl/AbstractGraphVisitorImpl.java | 60 ++--------
.../rx/impl/AllApplicationsObservable.java | 43 +++++++
.../rx/impl/AllApplicationsObservableImpl.java | 5 +-
.../rx/impl/AllEntitiesInSystemImpl.java | 14 +--
.../rx/impl/AllEntityIdsObservable.java | 51 +++++++++
.../rx/impl/AllEntityIdsObservableImpl.java | 87 ++++++++++++++
.../rx/impl/AllNodesInGraphImpl.java | 11 +-
.../corepersistence/rx/impl/EdgeScope.java | 51 +++++++++
.../corepersistence/util/CpNamingUtils.java | 15 +--
.../util/SerializableMapper.java | 89 +++++++++++++++
.../rx/ApplicationObservableTestIT.java | 3 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../core/rx/RxTaskSchedulerImpl.java | 9 +-
.../core/scope/ApplicationScope.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 ++++++
.../graph/serialization/EdgesObservable.java | 16 ++-
.../serialization/impl/EdgesObservableImpl.java | 23 +++-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 6 +-
.../persistence/index/CandidateResults.java | 2 +-
.../persistence/index/utils/StringUtils.java | 62 ----------
.../usergrid/rest/management/AdminUsersIT.java | 4 +-
32 files changed, 715 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
deleted file mode 100644
index 24f32f3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence;
-
-
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.entities.Application;
-
-import rx.Observable;
-
-
-/**
- * Interface for generating an observable of all ApplicationScope
- */
-public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 6ebff53..f49e066 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 22f599e..7a93606 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -23,8 +23,6 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
@@ -103,7 +101,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
//Remove all double indexes
- final Observable<IndexEdge> sourceScopes = edgesObservable.edgesFromSource( gm, entityId ).map(
+ final Observable<IndexEdge> sourceScopes = edgesObservable.edgesFromSourceAscending( gm, entityId ).map(
edge -> generateScopeToTarget( edge ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 77b4990..d00ef8e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.index;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
@@ -41,19 +43,25 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final MetricsFactory metricsFactory;
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
+ private final AllEntityIdsObservable allEntitiesObservable;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private AsyncIndexService asyncIndexService;
@Inject
- public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory, final
- MetricsFactory metricsFactory,
- final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
+ final MetricsFactory metricsFactory, final IndexService indexService,
+ final RxTaskScheduler rxTaskScheduler,
+ final AllEntityIdsObservable allEntitiesObservable,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.queryFig = queryFig;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
this.rxTaskScheduler = rxTaskScheduler;
+ this.allEntitiesObservable = allEntitiesObservable;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -76,7 +84,8 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, rxTaskScheduler );
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler,
+ entityCollectionManagerFactory );
case SQS:
return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
index 06310ae..8b5ced1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -26,13 +26,15 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
/**
* Low level queue service for indexing entities
*/
-public interface AsyncIndexService {
+public interface AsyncIndexService extends ReIndexService.IndexAction {
/**
@@ -43,4 +45,5 @@ public interface AsyncIndexService {
* @param entity The entity to index
*/
void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
index 3e2a271..0efb964 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -20,10 +20,19 @@
package org.apache.usergrid.corepersistence.index;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -33,16 +42,18 @@ import rx.Observable;
@Singleton
public class InMemoryAsyncIndexService implements AsyncIndexService {
+ private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncIndexService.class);
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@Inject
- public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.indexService = indexService;
-
-
this.rxTaskScheduler = rxTaskScheduler;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -53,7 +64,29 @@ public class InMemoryAsyncIndexService implements AsyncIndexService {
//only process the same version, otherwise ignore
Observable.just( toIndex ).doOnNext( entity -> {
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
indexService.indexEntity( applicationScope, entity );
} ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
}
+
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+
+ final Id entityId = entityIdScope.getId();
+
+ final Entity
+ entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
+ entityId ).toBlocking().lastOrDefault( null );
+
+
+ if(entity == null){
+ log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
+ }
+
+ indexService.indexEntity(applicationScope, entity );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
index a7d2450..82ed496 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
@@ -95,4 +95,9 @@ public interface QueryFig extends GuicyFig {
int getIndexQueueSize();
+ @Default("30000")
+ @Key("elasticsearch.reindex.sample.interval")
+ long getReIndexSampleInterval();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
new file mode 100644
index 0000000..dca6cac
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.Observer;
+
+
+/**
+ * An interface for re-indexing all entities in an application
+ */
+public interface ReIndexService {
+
+
+ /**
+ * Reindex all applications using the cursor provided
+ *
+ * @param startTimestamp The timestamp to start seeking from
+ *
+ * @return a cursor that can be used to resume the operation on the next run
+ */
+ IndexResponse reIndex( final rx.Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
+ final Optional<Long> startTimestamp, final IndexAction indexAction );
+
+
+ /**
+ * The response when requesting a re-index operation
+ */
+ class IndexResponse {
+ final String cursor;
+ final Observable<Long> indexedEdgecount;
+
+
+ public IndexResponse( final String cursor, final Observable<Long> indexedEdgecount ) {
+ this.cursor = cursor;
+ this.indexedEdgecount = indexedEdgecount;
+ }
+
+
+ /**
+ * Get the cursor used to resume this operation
+ * @return
+ */
+ public String getCursor() {
+ return cursor;
+ }
+
+
+ /**
+ * Return the observable long count of all edges indexed
+ * @return
+ */
+ public Observable<Long> getCount() {
+ return indexedEdgecount;
+ }
+ }
+
+
+
+
+ /**
+ * Callback to perform an index operation based on an scope during bulk re-index operations
+ */
+ @FunctionalInterface
+ interface IndexAction {
+
+ void index( final EntityIdScope entityIdScope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
new file mode 100644
index 0000000..5c022e1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.corepersistence.util.SerializableMapper;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.StringUtils;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.observables.ConnectableObservable;
+
+
+public class ReIndexServiceImpl implements ReIndexService {
+
+ private static final MapScope RESUME_MAP_SCOPTE =
+ new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "reindexresume" );
+
+ //Keep cursors to resume re-index for 1 day. This is far beyond it's useful real world implications anyway.
+ private static final int INDEX_TTL = 60 * 60 * 24 * 10;
+
+
+ private final AllEntityIdsObservable allEntityIdsObservable;
+ private final QueryFig queryFig;
+ private final RxTaskScheduler rxTaskScheduler;
+ private final MapManager mapManager;
+
+
+ public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
+ final MapManagerFactory mapManagerFactory, final QueryFig queryFig,
+ final RxTaskScheduler rxTaskScheduler ) {
+ this.allEntityIdsObservable = allEntityIdsObservable;
+ this.queryFig = queryFig;
+ this.rxTaskScheduler = rxTaskScheduler;
+
+ this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPTE );
+ }
+
+
+ @Override
+ public IndexResponse reIndex( final Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
+ final Optional<Long> startTimestamp, final IndexAction indexAction ) {
+
+
+ //load our last emitted Scope if a cursor is present
+ if ( cursor.isPresent() ) {
+ throw new UnsupportedOperationException( "Build this" );
+ }
+
+ final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ //create an observable that loads each entity and indexes it, start it running with publish
+ final ConnectableObservable<EdgeScope> runningReIndex =
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, startTimestamp )
+
+ //for each edge, create our scope and index on it
+ .doOnNext( edge -> indexAction
+ .index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) )
+
+ .subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).publish();
+
+
+ //count our longs
+ final Observable<Long> indexedCount = runningReIndex.countLong();
+
+
+ //start our sampler and state persistence
+ //take a sample every sample interval to allow us to resume state with minimal loss
+ runningReIndex.sample( queryFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
+ rxTaskScheduler.getAsyncIOScheduler() )
+ .doOnNext( edge -> {
+
+ final String serializedState = SerializableMapper.asString( edge );
+
+ mapManager.putString( newCursor, serializedState, INDEX_TTL );
+ } ).subscribe();
+
+
+ return new IndexResponse( newCursor, indexedCount );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
index dfcb97a..6d06637 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -233,6 +234,12 @@ public class SQSAsyncIndexService implements AsyncIndexService {
}
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ }
+
+
/**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
*/
@@ -258,6 +265,6 @@ public class SQSAsyncIndexService implements AsyncIndexService {
@Override
public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
-
+ throw new UnsupportedOperationException( "Implement me" );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
index ef66481..e307233 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -20,88 +20,46 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import rx.Observable;
-import rx.functions.Func1;
/**
- * An observable that will emit every entity Id stored in our entire system across all apps.
- * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
- * source node
+ * An observable that returns all entities in the collections
*/
public abstract class AbstractGraphVisitorImpl<T> implements MigrationDataProvider<T> {
private final AllApplicationsObservable applicationObservable;
- private final GraphManagerFactory graphManagerFactory;
- private final TargetIdObservable targetIdObservable;
+ private final AllEntityIdsObservable allEntityIdsObservable;
@Inject
public AbstractGraphVisitorImpl( AllApplicationsObservable applicationObservable,
- GraphManagerFactory graphManagerFactory, TargetIdObservable targetIdObservable ) {
+ final AllEntityIdsObservable allEntityIdsObservable ) {
this.applicationObservable = applicationObservable;
- this.graphManagerFactory = graphManagerFactory;
- this.targetIdObservable = targetIdObservable;
+ this.allEntityIdsObservable = allEntityIdsObservable;
}
@Override
public Observable<T> getData() {
- return applicationObservable.getData().flatMap( new Func1<ApplicationScope, Observable<T>>() {
- @Override
- public Observable<T> call( final ApplicationScope applicationScope ) {
- return getAllEntities( applicationScope );
- }
- } );
+ return allEntityIdsObservable.getEntities( applicationObservable.getData() ).map(
+ entityIdScope -> generateData( entityIdScope ) );
}
- private Observable<T> getAllEntities(final ApplicationScope applicationScope) {
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
- final Id applicationId = applicationScope.getApplication();
-
- //load all nodes that are targets of our application node. I.E.
- // entities that have been saved
- final Observable<Id> entityNodes =
- targetIdObservable.getTargetNodes(gm, applicationId);
-
- //emit Scope + ID
-
- //create our application node to emit since it's an entity as well
- final Observable<Id> applicationNode = Observable.just(applicationId);
-
- //merge both the specified application node and the entity node
- // so they all get used
- return Observable
- .merge( applicationNode, entityNodes ).
- map( new Func1<Id, T>() {
- @Override
- public T call( final Id id ) {
- return generateData(applicationScope, id);
- }
- } );
- }
-
-
/**
* Generate the data for the observable stream from the scope and the node id
- * @param applicationScope
- * @param nodeId
+ * @param entityIdScope
* @return
*/
- protected abstract T generateData(final ApplicationScope applicationScope, final Id nodeId);
+ protected abstract T generateData(final EntityIdScope entityIdScope);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
new file mode 100644
index 0000000..76ad8a1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.amazonaws.services.opsworks.model.App;
+
+import rx.Observable;
+
+
+/**
+ * Interface for generating an observable of all ApplicationScope
+ */
+public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
+
+
+ /**
+ * Get an observable of application scopes to be used
+ * @return
+ */
+ Observable<ApplicationScope> getData();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 5151857..5e2f6b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -23,13 +23,11 @@ package org.apache.usergrid.corepersistence.rx.impl;
import java.util.Arrays;
import java.util.UUID;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -40,6 +38,7 @@ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.utils.UUIDUtils;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
index 83f7d84..a315482 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,17 +40,14 @@ public class AllEntitiesInSystemImpl extends AbstractGraphVisitorImpl<EntityIdSc
@Inject
public AllEntitiesInSystemImpl( final AllApplicationsObservable applicationObservable,
- final GraphManagerFactory graphManagerFactory,
- final TargetIdObservable targetIdObservable ) {
- super( applicationObservable, graphManagerFactory, targetIdObservable );
+ final AllEntityIdsObservable allEntityIdsObservable) {
+ super( applicationObservable, allEntityIdsObservable );
}
@Override
- protected EntityIdScope generateData( final ApplicationScope applicationScope, final Id nodeId ) {
-
- final EntityIdScope idScope = new EntityIdScope( applicationScope, nodeId );
-
- return idScope;
+ protected EntityIdScope generateData( final EntityIdScope entityIdScope ) {
+ return entityIdScope;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
new file mode 100644
index 0000000..c805a59
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import com.google.common.base.Optional;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import rx.Observable;
+
+
+/**
+ * An implementation that will provide all entityId scopes in the system
+ */
+public interface AllEntityIdsObservable {
+
+ /**
+ * Return an observable of scopes from the given appScopes
+ * @param appScopes
+ * @return An observable of entityId scopes
+ */
+ Observable<EntityIdScope> getEntities( final Observable<ApplicationScope> appScopes );
+
+ /**
+ * Get all edges that represent edges to entities in the system
+ * @param appScopes
+ * @param startTime The time to
+ * @return
+ */
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<Long> startTime);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
new file mode 100644
index 0000000..1f74420
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+
+import com.google.common.base.Optional;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * An implementation that will provide all entityId scopes in the system
+ */
+public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
+ private final GraphManagerFactory graphManagerFactory;
+ private final TargetIdObservable targetIdObservable;
+ private final EdgesObservable edgesObservable;
+
+
+ public AllEntityIdsObservableImpl( final GraphManagerFactory graphManagerFactory,
+ final TargetIdObservable targetIdObservable,
+ final EdgesObservable edgesObservable ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.targetIdObservable = targetIdObservable;
+ this.edgesObservable = edgesObservable;
+ }
+
+
+ @Override
+ public Observable<EntityIdScope> getEntities( final Observable<ApplicationScope> appScopes ) {
+
+ return appScopes.flatMap( applicationScope -> {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final Id applicationId = applicationScope.getApplication();
+
+ //load all nodes that are targets of our application node. I.E.
+ // entities that have been saved
+ final Observable<Id> entityNodes = targetIdObservable.getTargetNodes( gm, applicationId );
+
+
+ //create our application node to emit since it's an entity as well
+ final Observable<Id> applicationNode = Observable.just( applicationId );
+
+ //merge both the specified application node and the entity node
+ // so they all get used
+ return Observable.merge( applicationNode, entityNodes ).
+ map( id -> new EntityIdScope( applicationScope, id ) );
+ } );
+ }
+
+
+ @Override
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<Long> startTime) {
+
+ return appScopes.flatMap( applicationScope -> {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+
+ return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(), startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
+ } );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
index 19292b8..f802654 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
@@ -42,14 +42,13 @@ public class AllNodesInGraphImpl extends AbstractGraphVisitorImpl<GraphNode> {
@Inject
public AllNodesInGraphImpl( final AllApplicationsObservable applicationObservable,
- final GraphManagerFactory graphManagerFactory,
- final TargetIdObservable targetIdObservable ) {
- super( applicationObservable, graphManagerFactory, targetIdObservable );
+ final AllEntityIdsObservable allEntityIdsObservable ) {
+ super( applicationObservable, allEntityIdsObservable );
}
@Override
- protected GraphNode generateData( final ApplicationScope applicationScope, final Id nodeId ) {
- return new GraphNode( applicationScope, nodeId );
+ protected GraphNode generateData( final EntityIdScope entityIdScope ) {
+ return new GraphNode(entityIdScope.getApplicationScope(), entityIdScope.getId());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
new file mode 100644
index 0000000..c5639ce
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+
+
+/**
+ * Get the scope for the edge
+ */
+public final class EdgeScope implements Serializable {
+ private final ApplicationScope applicationScope;
+ private final Edge edge;
+
+
+ public EdgeScope( final ApplicationScope applicationScope, final Edge edge ) {
+ this.applicationScope = applicationScope;
+ this.edge = edge;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+ public Edge getEdge() {
+ return edge;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index c60b86e..67cc0ca 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -73,13 +73,6 @@ public class CpNamingUtils {
public static String TYPES_BY_UUID_MAP = "zzz_typesbyuuid_zzz";
- /**
- * Create an Id object from the entity ref
- */
- public static Id createId( final EntityRef entityRef ) {
- return new SimpleId( entityRef.getUuid(), entityRef.getType() );
- }
-
/**
* Generate a standard edge name for our graph using the connection name. To be used only for searching. DO NOT use
@@ -229,6 +222,14 @@ public class CpNamingUtils {
/**
+ * Generate an application scope for the management application
+ * @return
+ */
+ public static Id getManagementApplicationId(){
+ return generateApplicationId( MANAGEMENT_APPLICATION_ID );
+ }
+
+ /**
* Get the map scope for the applicationId to store entity uuid to type mapping
*/
public static MapScope getEntityTypeMapScope( final Id applicationId ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
new file mode 100644
index 0000000..1ad4115
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.util;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A simple utility for serializing serializable classes to/and from strings. To be used for small object storage only, such as resume on re-index
+ * storing data such as entities should be specialized.
+ */
+public class SerializableMapper {
+
+ private static final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
+ static{
+ MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ }
+
+ /**
+ * Get value as a string
+ * @param toSerialize
+ * @param <T>
+ * @return
+ */
+ public static <T extends Serializable> String asString(final T toSerialize){
+ try {
+ return MAPPER.writeValueAsString( toSerialize );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new RuntimeException( "Unable to process json", e );
+ }
+ }
+
+
+ /**
+ * Write the value as a string
+ * @param serialized
+ * @param clazz
+ * @param <T>
+ * @return
+ */
+ public static <T extends Serializable> T fromString(final String serialized, final Class<T> clazz){
+ Preconditions.checkNotNull(serialized, "serialized string cannot be null");
+
+
+ InputStream stream = new ByteArrayInputStream(serialized.getBytes( StandardCharsets.UTF_8));
+
+ try {
+ return MAPPER.readValue( stream, clazz );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( String.format("Unable to parse string '%s'", serialized), e );
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
index 353e2fa..a619e4d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
@@ -28,8 +28,8 @@ import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
@@ -52,6 +52,7 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
final Application createdApplication = app.getEntityManager().getApplication();
+
AllApplicationsObservable applicationObservable =SpringResource.getInstance().getBean(Injector.class).getInstance(AllApplicationsObservable.class);
//now our get all apps we expect. There may be more, but we don't care about those.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
index 6d228b2..92f2b01 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.corepersistence.rx;
import java.util.HashSet;
import java.util.Set;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.junit.Test;
@@ -35,15 +34,12 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.utils.EdgeTestUtils;
import com.google.inject.Injector;
-import rx.functions.Action1;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -93,7 +89,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesFromSourceObservable.edgesFromSource( gm, applicationId ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceAscending( gm, applicationId ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
@@ -122,7 +118,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
//test connections
- edgesFromSourceObservable.edgesFromSource( gm, source).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceAscending( gm, source ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
index 219cde6..311f7f4 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
@@ -83,8 +83,7 @@ public class RxTaskSchedulerImpl implements RxTaskScheduler {
public MaxSizeThreadPool( final BlockingQueue<Runnable> queue, final int maxPoolSize ) {
- super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ),
- new RejectedHandler() );
+ super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ), new RejectedHandler() );
}
}
@@ -118,11 +117,11 @@ public class RxTaskSchedulerImpl implements RxTaskScheduler {
@Override
public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- log.warn( "{} task queue full, rejecting task {}", poolName, r );
+ log.warn( "{} task queue full, rejecting task {} and running in thread {}", poolName, r, Thread.currentThread().getName() );
- //TODO T.N. do we want to run this on the caller thread?
+ //We've decided we want to have a "caller runs" policy, to just invoke the task when rejected
- throw new RejectedExecutionException( "Unable to run task, queue full" );
+ r.run();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
index 920421d..5d22eff 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
@@ -19,13 +19,15 @@
package org.apache.usergrid.persistence.core.scope;
+import java.io.Serializable;
+
import org.apache.usergrid.persistence.model.entity.Id;
/**
* A scope used for organizations
*/
-public interface ApplicationScope {
+public interface ApplicationScope extends Serializable {
/**
* Get an Application scope
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
new file mode 100644
index 0000000..acfc2d8
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.core.util;
+
+
+import java.util.UUID;
+
+
+public class StringUtils extends org.apache.commons.lang.StringUtils {
+
+
+ /**
+ * Remove dashes from our uuid
+ * @param uuid
+ * @return
+ */
+ public static String sanitizeUUID(final UUID uuid){
+ return uuid.toString().replace( "-", "" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index fa73991..9e7b8e6 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -22,6 +22,9 @@ package org.apache.usergrid.persistence.graph.serialization;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
import rx.Observable;
/**
@@ -35,7 +38,18 @@ public interface EdgesObservable {
* @param sourceNode
* @return
*/
- Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode);
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode );
+
+
+ /**
+ * Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
+ * @param gm
+ * @param sourceNode
+ * @param startTimestamp
+ * @return
+ */
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ final Optional<Long> startTimestamp );
/**
* Get all edges from the source node with the target type
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 2264cbd..859ca2e 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -32,6 +32,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Func1;
@@ -53,7 +55,7 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode ) {
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode ) {
final Observable<String> edgeTypes =
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
@@ -69,6 +71,25 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ final Optional<Long> startTimestamp ) {
+
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+
+
+ return edgeTypes.flatMap( edgeType -> {
+
+ logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
+
+ return gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( sourceNode, edgeType, startTimestamp.or( Long.MIN_VALUE ), SearchByEdgeType.Order.ASCENDING,
+ null ) );
+ } );
+ }
+
+
+ @Override
public Observable<Edge> getEdgesFromSource( final GraphManager gm, final Id sourceNode, final String targetType ) {
final Observable<String> edgeTypes =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 9729e63..5cf5117 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -55,7 +55,7 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSource(gm, sourceNode ).map( new Func1<Edge, Id>() {
+ return edgesFromSourceObservable.edgesFromSourceAscending( gm, sourceNode ).map( new Func1<Edge, Id>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 2e0412f..0df26ff 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -38,8 +38,6 @@ import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSeri
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
import rx.schedulers.Schedulers;
import java.util.List;
@@ -87,9 +85,9 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
-
+
//get edges from the source
- return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
+ return edgesFromSourceObservable.edgesFromSourceAscending( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
final MutationBatch batch = keyspace.prepareMutationBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 14130bc..e4624a9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
+import static org.apache.usergrid.persistence.core.util.StringUtils.sanitizeUUID;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
deleted file mode 100644
index a567594..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.index.utils;
-
-
-import java.util.Arrays;
-import java.util.UUID;
-
-import org.apache.commons.io.IOUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.usergrid.persistence.index.utils.ConversionUtils.string;
-
-
-public class StringUtils extends org.apache.commons.lang.StringUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger( StringUtils.class );
-
-
-
- public static String stringOrSubstringBeforeFirst( String str, char c ) {
- if ( str == null ) {
- return null;
- }
- int i = str.indexOf( c );
- if ( i != -1 ) {
- return str.substring( 0, i );
- }
- return str;
- }
-
-
- public static String toString( Object obj ) {
- return string( obj );
- }
-
-
- /**
- * Remove dashes from our uuid
- * @param uuid
- * @return
- */
- public static String sanitizeUUID(final UUID uuid){
- return uuid.toString().replace( "-", "" );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
index 36a9400..de3a6ce 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
@@ -29,14 +29,13 @@ import javax.mail.internet.MimeMultipart;
import org.junit.Before;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import org.jvnet.mock_javamail.Mailbox;
import org.apache.usergrid.management.MockImapClient;
-import org.apache.usergrid.persistence.index.utils.StringUtils;
+import org.apache.usergrid.persistence.core.util.StringUtils;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
import org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt.ManagementResource;
@@ -46,7 +45,6 @@ import org.apache.usergrid.rest.test.resource2point0.model.Entity;
import org.apache.usergrid.rest.test.resource2point0.model.QueryParameters;
import org.apache.usergrid.rest.test.resource2point0.model.Token;
-import com.fasterxml.jackson.databind.JsonNode;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.representation.Form;
[03/16] incubator-usergrid git commit: Initial pass at moving queues
to core
Posted by sf...@apache.org.
Initial pass at moving queues to core
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/84d779fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/84d779fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/84d779fc
Branch: refs/heads/USERGRID-578
Commit: 84d779fc323b0f6c130c03a8982a39f5d5025b17
Parents: 8bb8a4f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Apr 15 14:21:29 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 11:58:28 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 8 +
.../usergrid/corepersistence/CoreModule.java | 44 ++-
.../corepersistence/CpEntityManager.java | 167 +++++-----
.../corepersistence/CpEntityManagerFactory.java | 18 +-
.../corepersistence/CpManagerCache.java | 3 +
.../corepersistence/CpRelationManager.java | 102 +++----
.../CpRelationManagerFactory.java | 46 ---
.../usergrid/corepersistence/GuiceFactory.java | 19 +-
.../events/EntityDeletedHandler.java | 5 +-
.../events/EntityVersionCreatedHandler.java | 8 +-
.../events/EntityVersionDeletedHandler.java | 19 +-
.../corepersistence/index/BufferQueue.java | 68 +++++
.../index/BufferQueueInMemoryImpl.java | 116 +++++++
.../index/BufferQueueSQSImpl.java | 306 +++++++++++++++++++
.../index/IndexQueueService.java | 45 +++
.../corepersistence/index/IndexService.java | 50 +++
.../corepersistence/index/IndexServiceImpl.java | 148 +++++++++
.../corepersistence/index/QueryFig.java | 98 ++++++
.../corepersistence/index/QueueProvider.java | 112 +++++++
.../index/SQSIndexQueueServiceImpl.java | 35 +++
.../corepersistence/util/CpNamingUtils.java | 4 +-
.../usergrid/persistence/EntityManager.java | 2 -
.../persistence/GuiceAdapterBeanFactory.java | 114 +++++++
.../usergrid/persistence/PersistenceModule.java | 162 ++++++++++
.../corepersistence/TestIndexModule.java | 35 +++
.../index/BufferQueueSQSImplTest.java | 179 +++++++++++
.../persistence/core/aws/NoAWSCredsRule.java | 98 ++++++
.../graph/serialization/EdgesObservable.java | 23 ++
.../serialization/impl/EdgesObservableImpl.java | 68 +++--
.../persistence/index/EntityIndexBatch.java | 3 +-
.../usergrid/persistence/index/IndexFig.java | 104 ++-----
.../persistence/index/guice/IndexModule.java | 5 +-
.../persistence/index/guice/QueueProvider.java | 116 -------
.../persistence/index/impl/BufferQueue.java | 66 ----
.../index/impl/BufferQueueInMemoryImpl.java | 115 -------
.../index/impl/BufferQueueSQSImpl.java | 306 -------------------
.../impl/EsApplicationEntityIndexImpl.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 16 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 50 +--
.../index/impl/EsIndexBufferConsumerImpl.java | 237 ++++++--------
.../index/impl/EsIndexBufferProducerImpl.java | 59 ----
.../index/impl/FailureMonitorImpl.java | 26 --
.../index/impl/FlushBufferQueue.java | 23 ++
.../index/impl/IndexBufferConsumer.java | 15 +-
.../index/impl/IndexBufferProducer.java | 32 --
.../persistence/index/impl/IndexIdentifier.java | 46 +++
.../index/impl/IndexIdentifierImpl.java | 118 +------
.../index/impl/IndexOperationMessage.java | 139 +++++++++
.../index/impl/IndexRefreshCommandImpl.java | 12 +-
.../index/migration/LegacyIndexIdentifier.java | 4 +-
.../index/guice/TestIndexModule.java | 22 +-
.../index/impl/BufferQueueSQSImplTest.java | 175 -----------
.../persistence/queue/NoAWSCredsRule.java | 98 ------
.../persistence/queue/QueueManagerTest.java | 3 +-
stack/services/pom.xml | 50 +--
.../notifications/NotifiersServiceIT.java | 19 +-
57 files changed, 2232 insertions(+), 1749 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 6f3c381..3504be4 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -163,6 +163,14 @@
<dependencies>
+
+ <!-- dependent on wiring guice and spring -->
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-spring</artifactId>
+ <version>4.0-beta5</version>
+ </dependency>
+
<!-- Apache Dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7859ffc..7e8af87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,7 +16,13 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.corepersistence.index.BufferQueue;
+import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.QueryFig;
+import org.apache.usergrid.corepersistence.index.QueueProvider;
import org.apache.usergrid.corepersistence.migration.*;
+import org.apache.usergrid.persistence.PersistenceModule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.safehaus.guicyfig.GuicyFigModule;
import org.springframework.context.ApplicationContext;
@@ -56,25 +62,18 @@ import com.google.inject.multibindings.Multibinder;
*/
public class CoreModule extends AbstractModule {
- /**
- * TODO this is a circular dependency, and should be refactored
- */
- private LazyEntityManagerFactoryProvider lazyEntityManagerFactoryProvider;
- public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
+ public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
- public CoreModule( final ApplicationContext context ) {
- this.lazyEntityManagerFactoryProvider = new LazyEntityManagerFactoryProvider( context );
- }
@Override
protected void configure() {
- //See TODO, this is fugly
- bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
+// //See TODO, this is fugly
+// bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
install( new CommonModule());
install( new CollectionModule() {
@@ -141,31 +140,24 @@ public class CoreModule extends AbstractModule {
plugins.addBinding().to( AppInfoMigrationPlugin.class );
plugins.addBinding().to( MigrationModuleVersionPlugin.class );
- bind(AllApplicationsObservable.class).to(AllApplicationsObservableImpl.class);
+ bind( AllApplicationsObservable.class ).to( AllApplicationsObservableImpl.class );
- install(new GuicyFigModule(ApplicationIdCacheFig.class));
-
- }
+ /*****
+ * Indexing service
+ *****/
- /**
- * TODO, this is a hack workaround due to the guice/spring EMF circular dependency
- * Once the entity managers have been refactored and moved into guice, remove this dependency.
- *
- */
- public static class LazyEntityManagerFactoryProvider implements Provider<EntityManagerFactory>{
- private final ApplicationContext context;
+ bind(IndexService.class).to(IndexServiceImpl.class);
+ //bind the queue provider
+ bind( BufferQueue.class).toProvider( QueueProvider.class );
- public LazyEntityManagerFactoryProvider( final ApplicationContext context ) {this.context = context;}
+ install( new GuicyFigModule( QueryFig.class ) );
+ install( new GuicyFigModule( ApplicationIdCacheFig.class ) );
- @Override
- public EntityManagerFactory get() {
- return this.context.getBean( EntityManagerFactory.class );
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2e7b7d8..a615a43 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -80,9 +80,6 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.map.MapManager;
@@ -125,7 +122,6 @@ import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
@@ -172,7 +168,6 @@ public class CpEntityManager implements EntityManager {
private UUID applicationId;
private Application application;
- private CpEntityManagerFactory emf;
private ManagerCache managerCache;
@@ -212,29 +207,34 @@ public class CpEntityManager implements EntityManager {
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
- public CpEntityManager() {
-
- }
-
- @Override
- public void init( EntityManagerFactory emf, UUID applicationId ) {
+ /**
+ * Fugly, make this part of DI
+ * @param cass
+ * @param counterUtils
+ * @param managerCache
+ * @param metricsFactory
+ * @param applicationId
+ */
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
- Preconditions.checkNotNull( emf, "emf must not be null" );
+ Preconditions.checkNotNull( cass, "cass must not be null" );
+ Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
+ Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
- this.emf = ( CpEntityManagerFactory ) emf;
- this.managerCache = this.emf.getManagerCache();
+
+ this.managerCache = managerCache;
this.applicationId = applicationId;
applicationScope = CpNamingUtils.getApplicationScope( applicationId );
ecm = managerCache.getEntityCollectionManager( applicationScope );
- this.cass = this.emf.getCassandraService();
- this.counterUtils = this.emf.getCounterUtils();
+ this.cass = cass;
+ this.counterUtils = counterUtils;
//Timer Setup
- this.metricsFactory = this.emf.getMetricsFactory();
+ this.metricsFactory = metricsFactory;
this.aggCounterTimer =this.metricsFactory.getTimer( CpEntityManager.class,
"cp.entity.get.aggregate.counters.timer" );
this.entCreateTimer =this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.create.timer" );
@@ -724,18 +724,18 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Map<String, Object> properties ) throws Exception {
- this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties);
+ this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties );
this.application = get( applicationId, Application.class );
}
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
- Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
- CpRelationManager rmi = CpRelationManagerFactory.get(
- this, emf, applicationId, entityRef, null, metricsFactory
- );
- return rmi;
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
+
+ CpRelationManager relationManager =
+ new CpRelationManager( metricsFactory, managerCache, this, applicationId, entityRef );
+ return relationManager;
}
@@ -819,7 +819,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
- return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias);
+ return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
}
@@ -848,7 +848,7 @@ public class CpEntityManager implements EntityManager {
new Object[] { ownerRef, collectionType, aliasValue } );
}
- return results.get(aliasValue);
+ return results.get( aliasValue );
}
@@ -1226,7 +1226,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF dictionaryCf = null;
- boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName);
+ boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName );
if ( entityHasDictionary ) {
dictionaryCf = ENTITY_DICTIONARIES;
@@ -1279,7 +1279,7 @@ public class CpEntityManager implements EntityManager {
Class<?> dictionaryCoType =
Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
- boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType);
+ boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
for ( int i = 0; i < elementNames.length; i++ ) {
@@ -1349,14 +1349,14 @@ public class CpEntityManager implements EntityManager {
@Override
public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isCollectionMember(collectionName, entity);
+ return getRelationManager( owner ).isCollectionMember( collectionName, entity );
}
@Override
public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isConnectionMember(connectionName, entity);
+ return getRelationManager( owner ).isConnectionMember( connectionName, entity );
}
@@ -1455,7 +1455,7 @@ public class CpEntityManager implements EntityManager {
public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
- return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef);
+ return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef );
}
@@ -1465,7 +1465,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( connectingEntity )
- .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
+ .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
}
@@ -1523,7 +1523,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities(connectionType, connectedEntityType, resultsLevel);
+ .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
}
@@ -1532,7 +1532,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities(connectionType, connectedEntityType, resultsLevel);
+ .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
}
@@ -1540,21 +1540,21 @@ public class CpEntityManager implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String entityType, Level level, int count ) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count);
+ return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
}
@Override
public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities(query);
+ return getRelationManager( connectingEntity ).searchConnectedEntities( query );
}
@Override
public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception {
- return getRelationManager( entity ).getConnectionIndexes(connectionType);
+ return getRelationManager( entity ).getConnectionIndexes( connectionType );
}
@@ -1799,8 +1799,8 @@ public class CpEntityManager implements EntityManager {
permission = permission.toLowerCase();
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp);
+ CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
@@ -1816,7 +1816,7 @@ public class CpEntityManager implements EntityManager {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, timestamp );
+ getRolePermissionsKey( groupId, roleName ), permission, timestamp );
//Adding graphite metrics
Timer.Context timeRevokeGroupRolePermission = entRevokeGroupPermissionTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -2281,7 +2281,7 @@ public class CpEntityManager implements EntityManager {
private MapManager getMapManagerForTypes() {
Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
MapManager mm = managerCache.getMapManager(ms);
@@ -2836,26 +2836,25 @@ public class CpEntityManager implements EntityManager {
CpWalker walker = new CpWalker( );
- walker.walkCollections(
- this, getApplication(), collectionName, reverse, new CpVisitor() {
+ walker.walkCollections( this, getApplication(), collectionName, reverse, new CpVisitor() {
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
+ @Override
+ public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- // swallow this, it just means this was already updated, which accomplishes our task
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
+ try {
+ em.update( entity );
+ po.onProgress( entity );
+ }
+ catch ( WriteOptimisticVerifyException wo ) {
+ // swallow this, it just means this was already updated, which accomplishes our task
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error repersisting entity", ex );
+ }
}
- }
- } );
+ } );
}
@@ -2890,32 +2889,34 @@ public class CpEntityManager implements EntityManager {
void indexEntityIntoCollection( final Edge edge, final org.apache.usergrid.persistence.model.entity.Entity collectionMember) {
- final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
- final EntityIndexBatch batch = aie.createBatch();
-
- // index member into entity collection | type scope
- final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
- batch.index( collectionIndexScope, collectionMember );
-
- //TODO REMOVE INDEX CODE
- // // index member into entity | all-types scope
- // IndexScope entityAllTypesScope = new IndexScopeImpl(
- // collectionEntity.getId(),
- // CpNamingUtils.ALL_TYPES, entityType );
- //
- // batch.index(entityAllTypesScope, memberEntity);
- //
- // // index member into application | all-types scope
- // IndexScope appAllTypesScope = new IndexScopeImpl(
- // getApplicationScope().getApplication(),
- // CpNamingUtils.ALL_TYPES, entityType );
- //
- // batch.index(appAllTypesScope, memberEntity);
-
- //Adding graphite metrics
- Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
- batch.execute();
- timeIndexEntityCollection.stop();
+ throw new UnsupportedOperationException( "Use the new interface!" );
+//
+// final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
+// final EntityIndexBatch batch = aie.createBatch();
+//
+// // index member into entity collection | type scope
+// final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
+// batch.index( collectionIndexScope, collectionMember );
+//
+// //TODO REMOVE INDEX CODE
+// // // index member into entity | all-types scope
+// // IndexScope entityAllTypesScope = new IndexScopeImpl(
+// // collectionEntity.getId(),
+// // CpNamingUtils.ALL_TYPES, entityType );
+// //
+// // batch.index(entityAllTypesScope, memberEntity);
+// //
+// // // index member into application | all-types scope
+// // IndexScope appAllTypesScope = new IndexScopeImpl(
+// // getApplicationScope().getApplication(),
+// // CpNamingUtils.ALL_TYPES, entityType );
+// //
+// // batch.index(appAllTypesScope, memberEntity);
+//
+// //Adding graphite metrics
+// Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
+// batch.execute();
+// timeIndexEntityCollection.stop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9907f91..e03ed47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -156,14 +156,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public ManagerCache getManagerCache() {
-
- if ( managerCache == null ) {
- managerCache = injector.getInstance( ManagerCache.class );
- }
- return managerCache;
- }
-
private Observable<EntityIdScope> getAllEntitiesObservable(){
return injector.getInstance( Key.get(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){})).getData();
}
@@ -184,16 +176,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager();
- em.init( this ,applicationId );
-
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, managerCache, metricsFactory, applicationId );
return em;
}
- public MetricsFactory getMetricsFactory(){
- return metricsFactory;
- }
-
@Override
public Entity createApplicationV2(String organizationName, String name) throws Exception {
return createApplicationV2( organizationName, name, null );
@@ -746,7 +732,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public Health getEntityStoreHealth() {
// could use any collection scope here, does not matter
- EntityCollectionManager ecm = getManagerCache().getEntityCollectionManager(
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager(
new ApplicationScopeImpl( new SimpleId( CpNamingUtils.MANAGEMENT_APPLICATION_ID, "application" ) ) );
return ecm.getHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index 4cae31e..f4fee0c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -30,6 +30,9 @@ import org.apache.usergrid.persistence.map.MapScope;
import com.google.inject.Inject;
+/**
+ * Cache for managing our other managers. Now just a delegate. Needs refactored away
+ */
public class CpManagerCache implements ManagerCache {
private final EntityCollectionManagerFactory ecmf;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1ae1db1..3c72b60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -107,9 +107,6 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
-
- private CpEntityManagerFactory emf;
-
private ManagerCache managerCache;
private EntityManager em;
@@ -126,25 +123,18 @@ public class CpRelationManager implements RelationManager {
private Timer updateCollectionTimer;
- public CpRelationManager() {}
-
-
- public CpRelationManager init( EntityManager em, CpEntityManagerFactory emf, UUID applicationId,
- EntityRef headEntity, IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory ) {
+ public CpRelationManager(final MetricsFactory metricsFactory, final ManagerCache managerCache, final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
Assert.notNull( em, "Entity manager cannot be null" );
- Assert.notNull( emf, "Entity manager factory cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
Assert.notNull( headEntity, "Head entity cannot be null" );
Assert.notNull( headEntity.getUuid(), "Head entity uuid cannot be null" );
// TODO: this assert should not be failing
//Assert.notNull( indexBucketLocator, "indexBucketLocator cannot be null" );
this.em = em;
- this.emf = emf;
this.applicationId = applicationId;
this.headEntity = headEntity;
- this.managerCache = emf.getManagerCache();
+ this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
this.metricsFactory = metricsFactory;
@@ -165,7 +155,7 @@ public class CpRelationManager implements RelationManager {
Assert.notNull( cpHeadEntity, String
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
- return this;
+
}
@@ -260,40 +250,42 @@ public class CpRelationManager implements RelationManager {
final org.apache.usergrid.persistence.model.entity.Entity cpEntity ) {
- final GraphManager gm = managerCache.getGraphManager( applicationScope );
-
- // loop through all types of edge to target
-
-
- final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
-
- final EntityIndexBatch entityIndexBatch = ei.createBatch();
-
- final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
-
- // for each edge type, emit all the edges of that type
- .flatMap( etype -> gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) ) )
-
- //for each edge we receive index and add to the batch
- .doOnNext( edge -> {
- // reindex the entity in the source entity's collection or connection index
-
- IndexEdge indexScope = generateScopeFromSource( edge );
-
- entityIndexBatch.index( indexScope, cpEntity );
-
- } ).doOnCompleted( () -> {
- Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
- entityIndexBatch.execute();
- timeElasticIndexBatch.stop();
- } ).count().toBlocking().lastOrDefault( 0 );
-
- //Adding graphite metrics
-
-
- logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
+ throw new UnsupportedOperationException( "Use the new interface" );
+
+// final GraphManager gm = managerCache.getGraphManager( applicationScope );
+//
+// // loop through all types of edge to target
+//
+//
+// final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+//
+// final EntityIndexBatch entityIndexBatch = ei.createBatch();
+//
+// final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
+//
+// // for each edge type, emit all the edges of that type
+// .flatMap( etype -> gm.loadEdgesToTarget(
+// new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
+// SearchByEdgeType.Order.DESCENDING, null ) ) )
+//
+// //for each edge we receive index and add to the batch
+// .doOnNext( edge -> {
+// // reindex the entity in the source entity's collection or connection index
+//
+// IndexEdge indexScope = generateScopeFromSource( edge );
+//
+// entityIndexBatch.index( indexScope, cpEntity );
+//
+// } ).doOnCompleted( () -> {
+// Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
+// entityIndexBatch.execute();
+// timeElasticIndexBatch.stop();
+// } ).count().toBlocking().lastOrDefault( 0 );
+//
+// //Adding graphite metrics
+//
+//
+// logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
}
@@ -494,9 +486,10 @@ public class CpRelationManager implements RelationManager {
// headEntityScope.getName()});
if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
- getRelationManager( itemEntity )
- .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
- getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
+ throw new UnsupportedOperationException( "Implement me directly in graph " );
+// getRelationManager( itemEntity )
+// .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
+// getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
}
return itemEntity;
@@ -555,7 +548,8 @@ public class CpRelationManager implements RelationManager {
addToCollection( collName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
- getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
+ throw new UnsupportedOperationException( "Implement me directly in graph " );
+// getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
}
}
@@ -1045,12 +1039,6 @@ public class CpRelationManager implements RelationManager {
}
- private CpRelationManager getRelationManager( EntityRef headEntity ) {
- CpRelationManager rmi = new CpRelationManager();
- rmi.init( em, emf, applicationId, headEntity, null, metricsFactory );
- return rmi;
- }
-
/** side effect: converts headEntity into an Entity if it is an EntityRef! */
private Entity getHeadEntity() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
deleted file mode 100644
index 4223f37..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.corepersistence;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-
-import java.util.UUID;
-
-/**
- * Factory to return and init relation manager instances
- */
-public class CpRelationManagerFactory {
-
-
- public static CpRelationManager get( EntityManager em,
- CpEntityManagerFactory emf,
- UUID applicationId,
- EntityRef headEntity,
- IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory){
- CpRelationManager relationManager = new CpRelationManager();
- relationManager.init(em,emf,applicationId,headEntity,indexBucketLocator,metricsFactory);
- return relationManager;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
index 566430f..3a08034 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
@@ -31,8 +31,7 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.PersistenceModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -47,7 +46,7 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator;
* Factory for configuring Guice then returning it
*/
@Singleton
-public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAware {
+public class GuiceFactory implements FactoryBean<Injector> {
private static final Logger logger = LoggerFactory.getLogger( GuiceFactory.class );
@@ -55,16 +54,16 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
private final Properties systemProperties;
-
private ApplicationContext applicationContext;
private Injector injector;
- public GuiceFactory( final CassandraHostConfigurator chc, final Properties systemProperties ) {
+ public GuiceFactory( final ApplicationContext applicationContext, final CassandraHostConfigurator chc, final Properties systemProperties ) {
this.chc = chc;
this.systemProperties = systemProperties;
+ this.applicationContext = applicationContext;
}
@@ -128,8 +127,10 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
throw new RuntimeException( "Fatal error loading configuration.", e );
}
- //this is seriously fugly, and needs removed
- injector = Guice.createInjector( new CoreModule( applicationContext ) );
+
+
+ //this is seriously fugly, and needs removed we shouldn't be mixing spring and guice
+ injector = Guice.createInjector( new CoreModule( ), new PersistenceModule( applicationContext ) );
return injector;
}
@@ -147,8 +148,4 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
}
- @Override
- public void setApplicationContext( final ApplicationContext applicationContext ) throws BeansException {
- this.applicationContext = applicationContext;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 6a37144..2de69ed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -42,11 +42,8 @@ public class EntityDeletedHandler implements EntityDeleted {
private static final Logger logger = LoggerFactory.getLogger( EntityDeletedHandler.class );
- private final EntityManagerFactory emf;
-
-
@Inject
- public EntityDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
+ public EntityDeletedHandler( ) {}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 0163fc2..20bdd55 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -41,15 +41,11 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
- private final EntityManagerFactory emf;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@Inject
- public EntityVersionCreatedHandler( final EntityManagerFactory emf,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.emf = emf;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ public EntityVersionCreatedHandler( ) {
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 700851a..22f599e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -30,9 +30,11 @@ import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -56,17 +58,20 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger( EntityVersionDeletedHandler.class );
- private final EntityManagerFactory emf;
private final EdgesObservable edgesObservable;
private final SerializationFig serializationFig;
+ private final EntityIndexFactory entityIndexFactory;
+ private final GraphManagerFactory graphManagerFactory;
@Inject
- public EntityVersionDeletedHandler( final EntityManagerFactory emf, final EdgesObservable edgesObservable,
- final SerializationFig serializationFig ) {
- this.emf = emf;
+ public EntityVersionDeletedHandler( final EdgesObservable edgesObservable, final SerializationFig serializationFig,
+ final EntityIndexFactory entityIndexFactory,
+ final GraphManagerFactory graphManagerFactory ) {
this.edgesObservable = edgesObservable;
this.serializationFig = serializationFig;
+ this.entityIndexFactory = entityIndexFactory;
+ this.graphManagerFactory = graphManagerFactory;
}
@@ -87,10 +92,8 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
} );
}
- CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf;
-
- final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
- final GraphManager gm = cpemf.getManagerCache().getGraphManager( scope );
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( scope );
+ final GraphManager gm = graphManagerFactory.createEdgeManager( scope );
//create an observable of all scopes to deIndex
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
new file mode 100644
index 0000000..fc1bdb7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+
+
+/**
+ * A temporary interface of our buffer Q to decouple of producer and consumer;
+ */
+public interface BufferQueue {
+
+ /**
+ * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
+ * Other queues may return the future after ack on the offer. See the implementation documentation for details.
+ * @param operation
+ */
+ public void offer(final IndexOperationMessage operation);
+
+
+ /**
+ * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
+ * May return less than the take size, but will never return null
+ *
+ * @param takeSize
+ * @param timeout
+ * @param timeUnit
+ * @return A null safe lid
+ */
+ public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
+
+
+ /**
+ * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
+ * This will set the future as done in in memory operations
+ *
+ * @param messages
+ */
+ public void ack(final List<IndexOperationMessage> messages);
+
+ /**
+ * Mark these message as failed. Set the exception in the future on local operation
+ *
+ * @param messages
+ */
+ public void fail(final List<IndexOperationMessage> messages, final Throwable t);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
new file mode 100644
index 0000000..0e43da3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class BufferQueueInMemoryImpl implements BufferQueue {
+
+
+ private final QueryFig fig;
+ private final ArrayBlockingQueue<IndexOperationMessage> messages;
+
+
+ @Inject
+ public BufferQueueInMemoryImpl( final QueryFig fig ) {
+ this.fig = fig;
+ messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+ try {
+ messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
+ }
+ catch ( InterruptedException e ) {
+ throw new RuntimeException("Unable to offer message to queue", e);
+ }
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
+ try {
+
+
+ messages.drainTo( response, takeSize );
+
+ //we got something, go process it
+ if ( response.size() > 0 ) {
+ return response;
+ }
+
+
+ final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+
+ if ( polled != null ) {
+ response.add( polled );
+
+ //try to add more
+ messages.drainTo( response, takeSize - 1 );
+ }
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+
+
+ return response;
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+ //if we have a future ack it
+ for ( final IndexOperationMessage op : messages ) {
+ op.done();
+ }
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+
+
+ for ( final IndexOperationMessage op : messages ) {
+ final BetterFuture<IndexOperationMessage> future = op.getFuture();
+
+ if ( future != null ) {
+ future.setError( t );
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
new file mode 100644
index 0000000..d955014
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
+ * performing
+ */
+@Singleton
+public class BufferQueueSQSImpl implements BufferQueue {
+
+ private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final MapManager mapManager;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope =
+ new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
+
+ this.mapManager = mapManagerFactory.createMapManager( scope );
+
+
+ this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+
+ //no op
+ if(operation.isEmpty()){
+ operation.getFuture().done();
+ return;
+ }
+
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //write to cassandra
+ this.mapManager.putString( identifier.toString(), payLoad, TTL );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ operation.done();
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+
+ final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if(messages.size() == 0){
+ return response;
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+ //look up the values
+ final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String key = getMessageKey( message );
+
+ //now see if the key was there
+ final String payload = storedCommands.get( key );
+
+ //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
+ // a DLQ
+
+ if ( payload == null ) {
+ continue;
+ }
+
+ final IndexOperationMessage messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexOperationMessage ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
+
+ //remove it from the map
+ mapManager.delete( key );
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+ //no op, just let it retry after the queue timeout
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexOperationMessage fromString( String s ) throws IOException {
+ IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexOperationMessage o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+ private String getMessageKey(final QueueMessage message){
+ return message.getBody().toString();
+ }
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexOperationMessage {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexOperationMessage source ) {
+ this.message = message;
+ this.addAllDeIndexRequest( source.getDeIndexRequests() );
+ this.addAllIndexRequest( source.getIndexRequests() );
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
new file mode 100644
index 0000000..8d7f222
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface IndexQueueService {
+
+
+ /**
+ * Queue an entity to be index asynchronously
+ * @param applicationScope
+ * @param entityId
+ * @param version
+ */
+ void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
new file mode 100644
index 0000000..2bf073c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Our low level indexing service operations
+ */
+public interface IndexService {
+
+
+ /**
+ * Perform an index update of the entity's state from Cassandra
+ *
+ * @param applicationScope The scope of the entity
+ * @param entity The entity
+ *
+ * @return An observable with the count of every
+ */
+ Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity );
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
new file mode 100644
index 0000000..2a7533a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeToTarget;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+
+
+/**
+ * Implementation of the indexing service
+ */
+@Singleton
+public class IndexServiceImpl implements IndexService {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
+ private final EdgesObservable edgesObservable;
+ private final IndexFig indexFig;
+
+
+ @Inject
+ public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory,
+ final EdgesObservable edgesObservable, IndexFig indexFig ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
+ this.edgesObservable = edgesObservable;
+ this.indexFig = indexFig;
+ }
+
+
+ @Override
+ public Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
+
+
+ //bootstrap the lower modules from their caches
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+
+ final Id entityId = entity.getId();
+
+
+ //we always index in the target scope
+ final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
+
+ //we may have to index
+ final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeToTarget( edge ) );
+
+
+ //we might or might not need to index from target-> source
+
+
+ final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
+
+
+ final Observable<IndexOperationMessage> observable =
+ //try to send a whole batch if we can
+ Observable.merge( sourceEdgesToIndex, targetSizes ).buffer( indexFig.getIndexBatchSize() )
+
+ //map into batches based on our buffer size
+ .flatMap( buffer -> Observable.from( buffer ).collect( () -> ei.createBatch(),
+ ( batch, indexEdge ) -> batch.index( indexEdge, entity ) )
+ //return the future from the batch execution
+ .flatMap( batch -> Observable.from( batch.execute() ) ) );
+
+ observable.toBlocking().last();
+
+
+ return Observable.just( 0 );
+ }
+
+
+ /**
+ * Get index edgs to the target
+ *
+ * @param graphManager The graph manager
+ * @param entityId The entitie's id
+ */
+ private Observable<IndexEdge> getIndexEdgesToTarget( final GraphManager graphManager, final Id entityId ) {
+
+ final String collectionName = InflectionUtils.pluralize( entityId.getType() );
+
+
+ final CollectionInfo collection = getDefaultSchema().getCollection( Application.ENTITY_TYPE, collectionName );
+
+ //nothing to do
+ if ( collection == null ) {
+ return Observable.empty();
+ }
+
+
+ final String linkedCollection = collection.getLinkedCollection();
+
+ /**
+ * Nothing to link
+ */
+ if ( linkedCollection == null ) {
+ return Observable.empty();
+ }
+
+
+ /**
+ * An observable of sizes as we execute batches
+ */
+ return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
+ .map( edge -> generateScopeFromSource( edge ) );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
new file mode 100644
index 0000000..a7d2450
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ * Application id cache fig
+ */
+public interface QueryFig extends GuicyFig {
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+
+ /**
+ * The number of worker threads to consume from the queue
+ */
+ public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
+
+ /**
+ * Amount of time to wait when reading from the queue
+ */
+ public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
+
+ /**
+ * Amount of time to wait when reading from the queue in milliseconds
+ */
+ public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
+
+
+ String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
+
+
+ @Default( "1000" )
+ @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
+ long getFailureRetryTime();
+
+ //give us 60 seconds to process the message
+ @Default( "60" )
+ @Key( INDEX_QUEUE_READ_TIMEOUT )
+ int getIndexQueueTimeout();
+
+ @Default( "2" )
+ @Key( ELASTICSEARCH_WORKER_COUNT )
+ int getWorkerCount();
+
+ @Default( "LOCAL" )
+ @Key( ELASTICSEARCH_QUEUE_IMPL )
+ String getQueueImplementation();
+
+ @Default( "1000" )
+ @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
+ long getQueueOfferTimeout();
+
+ /**
+ * size of the buffer to build up before you send results
+ */
+ @Default( "1000" )
+ @Key( INDEX_QUEUE_SIZE )
+ int getIndexQueueSize();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
new file mode 100644
index 0000000..d3920db
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class QueueProvider implements Provider<BufferQueue> {
+
+ private final QueryFig queryFig;
+
+ private final QueueManagerFactory queueManagerFactory;
+ private final MapManagerFactory mapManagerFactory;
+ private final MetricsFactory metricsFactory;
+
+ private BufferQueue bufferQueue;
+
+
+ @Inject
+ public QueueProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ this.queryFig = queryFig;
+
+
+ this.queueManagerFactory = queueManagerFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.metricsFactory = metricsFactory;
+ }
+
+
+ @Override
+ @Singleton
+ public BufferQueue get() {
+ if ( bufferQueue == null ) {
+ bufferQueue = getQueue();
+ }
+
+
+ return bufferQueue;
+ }
+
+
+ private BufferQueue getQueue() {
+ final String value = queryFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new BufferQueueInMemoryImpl( queryFig );
+ case SQS:
+ return new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
new file mode 100644
index 0000000..42f36b1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class SQSIndexQueueServiceImpl implements IndexQueueService {
+
+ @Override
+ public void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version ) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 0376780..c60b86e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -157,7 +157,7 @@ public class CpNamingUtils {
*/
public static SearchEdge createCollectionSearchEdge( final Id sourceId, final String connectionType ) {
return new SearchEdgeImpl( sourceId, getEdgeTypeFromCollectionName( connectionType ),
- SearchEdge.NodeType.SOURCE );
+ SearchEdge.NodeType.TARGET );
}
@@ -181,7 +181,7 @@ public class CpNamingUtils {
*/
public static SearchEdge createConnectionSearchEdge( final Id sourceId, final String connectionType ) {
return new SearchEdgeImpl( sourceId, getEdgeTypeFromConnectionType( connectionType ),
- SearchEdge.NodeType.SOURCE );
+ SearchEdge.NodeType.TARGET );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 7929b12..fc8b3d5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -689,8 +689,6 @@ public interface EntityManager {
/** @return the cass */
CassandraService getCass();
- public void init( EntityManagerFactory emf, UUID applicationId);
-
/** For testing purposes */
public void flushManagerCaches();
[16/16] incubator-usergrid git commit: merge
Posted by sf...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a0c2651e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a0c2651e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a0c2651e
Branch: refs/heads/USERGRID-578
Commit: a0c2651e31956c7a38f39893a44a6d478d764277
Parents: f130b7c 3480a36
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 09:55:02 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 09:55:02 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
stack/core/pom.xml | 8 +
.../AllApplicationsObservable.java | 35 ---
.../usergrid/corepersistence/CoreModule.java | 56 ++--
.../corepersistence/CpEntityManager.java | 273 ++++++-----------
.../corepersistence/CpEntityManagerFactory.java | 141 +++------
.../corepersistence/CpManagerCache.java | 3 +
.../corepersistence/CpRelationManager.java | 93 ++----
.../CpRelationManagerFactory.java | 46 ---
.../usergrid/corepersistence/GuiceFactory.java | 75 ++---
.../events/EntityDeletedHandler.java | 5 +-
.../events/EntityVersionCreatedHandler.java | 8 +-
.../events/EntityVersionDeletedHandler.java | 23 +-
.../index/AsyncIndexProvider.java | 122 ++++++++
.../index/AsyncReIndexService.java | 42 +++
.../index/InMemoryAsyncReIndexService.java | 89 ++++++
.../corepersistence/index/IndexEntityEvent.java | 75 +++++
.../corepersistence/index/IndexService.java | 52 ++++
.../corepersistence/index/IndexServiceImpl.java | 152 +++++++++
.../corepersistence/index/QueryFig.java | 103 +++++++
.../corepersistence/index/ReIndexAction.java | 33 ++
.../corepersistence/index/ReIndexService.java | 89 ++++++
.../index/ReIndexServiceImpl.java | 129 ++++++++
.../index/SQSAsyncReIndexService.java | 269 ++++++++++++++++
.../rx/impl/AbstractGraphVisitorImpl.java | 60 +---
.../rx/impl/AllApplicationsObservable.java | 43 +++
.../rx/impl/AllApplicationsObservableImpl.java | 5 +-
.../rx/impl/AllEntitiesInSystemImpl.java | 14 +-
.../rx/impl/AllEntityIdsObservable.java | 52 ++++
.../rx/impl/AllEntityIdsObservableImpl.java | 92 ++++++
.../rx/impl/AllNodesInGraphImpl.java | 11 +-
.../corepersistence/rx/impl/EdgeScope.java | 51 ++++
.../corepersistence/util/CpNamingUtils.java | 25 +-
.../util/SerializableMapper.java | 89 ++++++
.../usergrid/persistence/EntityManager.java | 7 -
.../persistence/EntityManagerFactory.java | 11 +-
.../persistence/GuiceAdapterBeanFactory.java | 114 +++++++
.../usergrid/persistence/PersistenceModule.java | 66 ++++
.../main/resources/usergrid-core-context.xml | 28 +-
.../corepersistence/TestIndexModule.java | 35 +++
.../corepersistence/index/PublishRxtest.java | 61 ++++
.../index/SQSAsyncIndexServiceTest.java | 167 ++++++++++
.../rx/ApplicationObservableTestIT.java | 3 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../usergrid/persistence/EntityManagerIT.java | 8 +-
.../PerformanceEntityRebuildIndexTest.java | 18 +-
.../cassandra/EntityManagerFactoryImplIT.java | 15 +-
.../resources/usergrid-custom-test.properties | 2 -
.../EntityCollectionManagerFactoryImpl.java | 15 +-
.../impl/EntityCollectionManagerImpl.java | 38 ++-
.../mvcc/stage/write/RollbackAction.java | 2 -
.../persistence/core/guice/CommonModule.java | 12 +
.../persistence/core/rx/RxSchedulerFig.java | 60 ++++
.../persistence/core/rx/RxTaskScheduler.java | 40 +++
.../core/rx/RxTaskSchedulerImpl.java | 128 ++++++++
.../core/scope/ApplicationScope.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 +++
.../persistence/core/aws/NoAWSCredsRule.java | 98 ++++++
.../graph/serialization/EdgesObservable.java | 40 ++-
.../serialization/impl/EdgesObservableImpl.java | 91 ++++--
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 6 +-
.../persistence/index/CandidateResults.java | 1 +
.../persistence/index/EntityIndexBatch.java | 3 +-
.../usergrid/persistence/index/IndexFig.java | 115 +++----
.../persistence/index/guice/IndexModule.java | 5 +-
.../persistence/index/guice/QueueProvider.java | 116 -------
.../persistence/index/impl/BufferQueue.java | 66 ----
.../index/impl/BufferQueueInMemoryImpl.java | 115 -------
.../index/impl/BufferQueueSQSImpl.java | 306 -------------------
.../impl/EsApplicationEntityIndexImpl.java | 16 +-
.../index/impl/EsEntityIndexBatchImpl.java | 16 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 52 +---
.../index/impl/EsIndexBufferConsumerImpl.java | 239 ++++++---------
.../index/impl/EsIndexBufferProducerImpl.java | 59 ----
.../index/impl/FailureMonitorImpl.java | 26 --
.../index/impl/FlushBufferQueue.java | 23 ++
.../index/impl/IndexBufferConsumer.java | 15 +-
.../index/impl/IndexBufferProducer.java | 32 --
.../persistence/index/impl/IndexIdentifier.java | 46 +++
.../index/impl/IndexIdentifierImpl.java | 118 +------
.../index/impl/IndexOperationMessage.java | 139 +++++++++
.../index/impl/IndexRefreshCommandImpl.java | 30 +-
.../persistence/index/impl/IndexingUtils.java | 6 +-
.../index/migration/LegacyIndexIdentifier.java | 4 +-
.../persistence/index/utils/StringUtils.java | 62 ----
.../persistence/index/usergrid-mappings.json | 4 +-
.../index/guice/TestIndexModule.java | 22 +-
.../index/impl/BufferQueueSQSImplTest.java | 175 -----------
.../persistence/queue/NoAWSCredsRule.java | 98 ------
.../persistence/queue/QueueManagerTest.java | 3 +-
.../usergrid/rest/management/AdminUsersIT.java | 4 +-
stack/services/pom.xml | 50 +--
.../notifications/NotifiersServiceIT.java | 19 +-
95 files changed, 3205 insertions(+), 2267 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a0c2651e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 716c7d8,e4624a9..97ac813
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@@ -27,10 -27,14 +27,11 @@@ package org.apache.usergrid.persistence
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import com.google.common.base.Optional;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import static org.apache.usergrid.persistence.core.util.StringUtils.sanitizeUUID;
++import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
/**
* Internal results class, should not be returned as results to a user.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a0c2651e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 672f090,43b55d0..89d6da7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@@ -23,13 -23,8 +23,12 @@@ package org.apache.usergrid.persistence
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
- import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
[08/16] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-541
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-541
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/078666ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/078666ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/078666ec
Branch: refs/heads/USERGRID-578
Commit: 078666ec10904bc5eea6708882d4e20df4d66f83
Parents: 9b9a1b0 8d965b3
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 10:38:01 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 10:38:01 2015 -0600
----------------------------------------------------------------------
.gitattributes | 17 +
.gitignore | 4 +
.usergridversion | 1 +
CHANGELOG | 31 +
NOTICE | 17 +-
docs/.gitignore | 2 +
docs/Makefile | 177 +
docs/README | 31 +
docs/_theme/sphinx_rtd_theme/__init__.py | 17 +
docs/_theme/sphinx_rtd_theme/breadcrumbs.html | 19 +
docs/_theme/sphinx_rtd_theme/footer.html | 33 +
docs/_theme/sphinx_rtd_theme/layout.html | 162 +
docs/_theme/sphinx_rtd_theme/layout_old.html | 205 +
docs/_theme/sphinx_rtd_theme/search.html | 50 +
docs/_theme/sphinx_rtd_theme/searchbox.html | 7 +
.../sphinx_rtd_theme/static/css/badge_only.css | 2 +
.../sphinx_rtd_theme/static/css/theme.css | 5 +
.../static/fonts/FontAwesome.otf | Bin 0 -> 62856 bytes
.../static/fonts/fontawesome-webfont.eot | Bin 0 -> 38205 bytes
.../static/fonts/fontawesome-webfont.svg | 414 +
.../static/fonts/fontawesome-webfont.ttf | Bin 0 -> 80652 bytes
.../static/fonts/fontawesome-webfont.woff | Bin 0 -> 44432 bytes
docs/_theme/sphinx_rtd_theme/static/js/theme.js | 47 +
docs/_theme/sphinx_rtd_theme/theme.conf | 8 +
docs/_theme/sphinx_rtd_theme/versions.html | 37 +
docs/concepts/activity.md | 1121 ++
docs/concepts/applications.md | 78 +
docs/concepts/assets.md | 20 +
docs/concepts/collections.md | 21 +
docs/concepts/events-and-counters.md | 268 +
docs/concepts/groups.md | 38 +
docs/concepts/query-language.md | 427 +
docs/concepts/relationships.md | 100 +
docs/concepts/roles-and-permissions.md | 18 +
docs/concepts/users-devices.md | 108 +
docs/conf.py | 273 +
docs/contribute-code.md | 24 +
docs/deploy-local.md | 179 +
docs/getting-up-and-running-locally.md | 72 +
docs/index.md | 12 +
docs/lib/sphinxcontrib_markdown.py | 58 +
docs/make.bat | 242 +
docs/organizations-admins.md | 55 +
docs/presentations-and-videos.md | 7 +
docs/presentations-and-videos/presentations.md | 21 +
docs/presentations-and-videos/videos.md | 11 +
docs/upgrading.md | 3 +
docs/usage.md | 16 +
portal/Gruntfile.js | 20 +-
portal/README.md | 35 +-
portal/build.sh | 1 +
portal/config.js | 3 +-
portal/css/main.css | 23 +-
portal/img/sdk-sprites-large.png | Bin 14642 -> 15115 bytes
portal/img/sdk-sprites.png | Bin 5027 -> 4401 bytes
portal/index-template.html | 1 +
portal/js/global/help-service.js | 2 +-
portal/js/global/ug-service.js | 5 +-
portal/package.json | 2 +-
portal/pom.xml | 98 +
release/README.txt | 43 +
release/changelog.rb | 150 +
release/pom.xml | 65 -
release/release-candidate.sh | 296 +
release/release.sh | 248 +
release/sign-release.sh | 30 -
release/src/main/assembly/source.xml | 57 -
sdks/android/.gitignore | 20 +
sdks/android/NOTICE | 12 +
sdks/android/README.md | 33 +
sdks/android/README.txt | 2 -
sdks/android/assembly.xml | 55 +
sdks/android/build_release_zip.sh | 10 +
sdks/android/pom.xml | 76 +-
.../apache/usergrid/android/client/Client.java | 1036 --
.../client/callbacks/ApiResponseCallback.java | 26 -
.../client/callbacks/ClientAsyncTask.java | 54 -
.../client/callbacks/ClientCallback.java | 25 -
.../callbacks/DeviceRegistrationCallback.java | 25 -
.../callbacks/GroupsRetrievedCallback.java | 29 -
.../client/callbacks/QueryResultsCallback.java | 25 -
.../android/client/utils/DeviceUuidFactory.java | 170 -
.../android/client/utils/ObjectUtils.java | 36 -
.../usergrid/android/sdk/CounterIncrement.java | 72 +
.../sdk/DefaultURLConnectionFactory.java | 36 +
.../apache/usergrid/android/sdk/UGClient.java | 3179 +++++
.../android/sdk/URLConnectionFactory.java | 30 +
.../sdk/callbacks/ApiResponseCallback.java | 31 +
.../android/sdk/callbacks/ClientAsyncTask.java | 66 +
.../android/sdk/callbacks/ClientCallback.java | 31 +
.../sdk/callbacks/GroupsRetrievedCallback.java | 35 +
.../sdk/callbacks/QueryResultsCallback.java | 33 +
.../usergrid/android/sdk/entities/Activity.java | 1019 ++
.../android/sdk/entities/Collection.java | 338 +
.../usergrid/android/sdk/entities/Device.java | 122 +
.../usergrid/android/sdk/entities/Entity.java | 552 +
.../usergrid/android/sdk/entities/Group.java | 151 +
.../usergrid/android/sdk/entities/Message.java | 159 +
.../usergrid/android/sdk/entities/User.java | 315 +
.../android/sdk/exception/ClientException.java | 43 +
.../android/sdk/response/AggregateCounter.java | 58 +
.../sdk/response/AggregateCounterSet.java | 117 +
.../android/sdk/response/ApiResponse.java | 774 ++
.../sdk/response/ClientCredentialsInfo.java | 64 +
.../android/sdk/response/QueueInfo.java | 47 +
.../android/sdk/utils/DeviceUuidFactory.java | 173 +
.../usergrid/android/sdk/utils/JsonUtils.java | 185 +
.../usergrid/android/sdk/utils/MapUtils.java | 42 +
.../usergrid/android/sdk/utils/ObjectUtils.java | 39 +
.../usergrid/android/sdk/utils/UrlUtils.java | 127 +
sdks/dotnet/NOTICE | 56 +
.../NSubstitute.1.6.0.0.nupkg | Bin 619760 -> 0 bytes
.../NSubstitute.1.6.0.0.nuspec | 16 -
.../packages/NSubstitute.1.6.0.0/README.txt | 0
.../NSubstitute.1.6.0.0/acknowledgements.txt | 14 -
.../lib/NET35/NSubstitute.XML | 432 -
.../lib/NET35/NSubstitute.dll | Bin 1039360 -> 0 bytes
.../lib/NET40/NSubstitute.XML | 432 -
.../lib/NET40/NSubstitute.dll | Bin 561152 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nupkg | Bin 94702 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nuspec | 16 -
sdks/dotnet/packages/NUnit.2.6.2/README.txt | 3 -
.../NUnit.2.6.2/lib/nunit.framework.dll | Bin 147456 -> 0 bytes
.../NUnit.2.6.2/lib/nunit.framework.xml | 10915 -----------------
.../packages/Newtonsoft.Json.4.5.11/LICENSE.txt | 21 +
.../Newtonsoft.Json.4.5.11.nupkg | Bin 1409718 -> 0 bytes
.../Newtonsoft.Json.4.5.11.nuspec | 16 -
.../packages/Newtonsoft.Json.4.5.11/README.txt | 3 -
.../lib/net20/Newtonsoft.Json.dll | Bin 406016 -> 0 bytes
.../lib/net20/Newtonsoft.Json.xml | 8542 -------------
.../lib/net35/Newtonsoft.Json.dll | Bin 366592 -> 0 bytes
.../lib/net35/Newtonsoft.Json.xml | 7678 ------------
.../lib/net40/Newtonsoft.Json.dll | Bin 391680 -> 0 bytes
.../lib/net40/Newtonsoft.Json.xml | 7921 ------------
.../Newtonsoft.Json.dll | Bin 315904 -> 0 bytes
.../Newtonsoft.Json.xml | 7107 -----------
.../lib/sl3-wp/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl3-wp/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4-windowsphone71/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl4-windowsphone71/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4/Newtonsoft.Json.dll | Bin 337408 -> 0 bytes
.../lib/sl4/Newtonsoft.Json.xml | 7250 -----------
.../lib/winrt45/Newtonsoft.Json.dll | Bin 367616 -> 0 bytes
.../lib/winrt45/Newtonsoft.Json.xml | 7446 -----------
.../dotnet/packages/RestSharp.104.1/LICENSE.txt | 202 +
.../RestSharp.104.1/RestSharp.104.1.nupkg | Bin 605731 -> 0 bytes
.../RestSharp.104.1/RestSharp.104.1.nuspec | 16 -
.../lib/net35-client/RestSharp.dll | Bin 146944 -> 0 bytes
.../lib/net35-client/RestSharp.xml | 2627 ----
.../RestSharp.104.1/lib/net35/RestSharp.dll | Bin 146944 -> 0 bytes
.../RestSharp.104.1/lib/net35/RestSharp.xml | 2627 ----
.../lib/net4-client/RestSharp.dll | Bin 150016 -> 0 bytes
.../lib/net4-client/RestSharp.xml | 2696 ----
.../RestSharp.104.1/lib/net4/RestSharp.dll | Bin 150016 -> 0 bytes
.../RestSharp.104.1/lib/net4/RestSharp.xml | 2696 ----
.../lib/sl4-wp71/RestSharp.WindowsPhone.dll | Bin 165376 -> 0 bytes
.../lib/sl4-wp71/RestSharp.WindowsPhone.xml | 3510 ------
.../lib/sl4/RestSharp.Silverlight.dll | Bin 124928 -> 0 bytes
.../lib/sl4/RestSharp.Silverlight.xml | 2445 ----
sdks/html5-javascript/lib/modules/Client.js | 29 +-
sdks/nodejs/lib/usergrid.js | 31 +-
sdks/php/lib/vendor/Apache/Usergrid/Client.php | 33 +-
sdks/php5/apache-usergrid/.gitignore | 12 +
.../Examples/attrubites/attributes.php | 122 +
.../Examples/collections/books.php | 130 +
.../Examples/collections/data.php | 38 +
.../Examples/collections/users.php | 120 +
sdks/php5/apache-usergrid/Examples/examples.md | 17 +
sdks/php5/apache-usergrid/Examples/examples.php | 245 +
.../Examples/management/management.php | 91 +
.../apache-usergrid/Examples/messages/data.php | 17 +
.../Examples/messages/messages.php | 62 +
.../Examples/notifications/data.php | 17 +
.../Examples/notifications/notifications.php | 86 +
.../Examples/notifications/notifiers.php | 79 +
sdks/php5/apache-usergrid/LICENSE | 202 +
sdks/php5/apache-usergrid/README.md | 219 +
sdks/php5/apache-usergrid/TODO.md | 2 +
sdks/php5/apache-usergrid/composer.json | 40 +
sdks/php5/apache-usergrid/composer.lock | 1225 ++
sdks/php5/apache-usergrid/phpunit.xml | 29 +
.../src/Api/Exception/BadRequestException.php | 34 +
.../src/Api/Exception/InvalidIdException.php | 33 +
.../src/Api/Exception/NotFoundException.php | 33 +
.../src/Api/Exception/ServerErrorException.php | 33 +
.../src/Api/Exception/UnauthorizedException.php | 34 +
.../src/Api/Exception/UsergridException.php | 152 +
.../apache-usergrid/src/Api/Filters/Boolean.php | 44 +
.../apache-usergrid/src/Api/Filters/Date.php | 44 +
.../apache-usergrid/src/Api/Filters/Number.php | 48 +
.../apache-usergrid/src/Api/GuzzleClient.php | 64 +
.../src/Api/Models/Application.php | 35 +
.../src/Api/Models/BaseCollection.php | 116 +
.../src/Api/Models/Collection.php | 37 +
.../apache-usergrid/src/Api/Models/Device.php | 50 +
.../apache-usergrid/src/Api/Models/Entity.php | 36 +
.../apache-usergrid/src/Api/Models/Event.php | 35 +
.../apache-usergrid/src/Api/Models/Group.php | 54 +
.../src/Api/Models/GuzzleCommandTrait.php | 59 +
.../apache-usergrid/src/Api/Models/Notifier.php | 36 +
.../src/Api/Models/Organization.php | 38 +
.../apache-usergrid/src/Api/Models/Receipt.php | 36 +
.../apache-usergrid/src/Api/Models/Role.php | 54 +
.../apache-usergrid/src/Api/Models/User.php | 77 +
.../apache-usergrid/src/Api/QueryAggregator.php | 61 +
.../src/Api/ResourceIterator.php | 69 +
sdks/php5/apache-usergrid/src/Api/Usergrid.php | 452 +
.../Oauth2/GrantType/AuthorizationCode.php | 77 +
.../Oauth2/GrantType/ClientCredentials.php | 73 +
.../Oauth2/GrantType/GrantTypeInterface.php | 41 +
.../GrantType/HWIOAuthBundleRefreshToken.php | 62 +
.../Oauth2/GrantType/PasswordCredentials.php | 75 +
.../Plugin/Oauth2/GrantType/RefreshToken.php | 74 +
.../src/Guzzle/Plugin/Oauth2/Oauth2Plugin.php | 212 +
.../Laravel/ApacheUsergridServiceProvider.php | 159 +
.../src/Laravel/Facades/Usergrid.php | 42 +
.../src/Manifests/1.0.0/Application.php | 979 ++
.../src/Manifests/1.0.0/Errors.php | 53 +
.../src/Manifests/1.0.0/Management.php | 1076 ++
.../src/Manifests/1.0.0/Manifest.php | 25 +
.../src/Manifests/1.0.0/Notification.php | 154 +
.../src/Manifests/1.0.1/Activities.php | 24 +
.../src/Manifests/1.0.1/Application.php | 1108 ++
.../src/Manifests/1.0.1/Assets.php | 24 +
.../src/Manifests/1.0.1/Custom.php | 381 +
.../src/Manifests/1.0.1/Devices.php | 465 +
.../src/Manifests/1.0.1/Errors.php | 50 +
.../src/Manifests/1.0.1/Events.php | 24 +
.../src/Manifests/1.0.1/Groups.php | 486 +
.../src/Manifests/1.0.1/Management.php | 1076 ++
.../src/Manifests/1.0.1/Manifest.php | 24 +
.../src/Manifests/1.0.1/Messages.php | 19 +
.../src/Manifests/1.0.1/Notifications.php | 241 +
.../src/Manifests/1.0.1/Notifiers.php | 166 +
.../src/Manifests/1.0.1/Receipts.php | 24 +
.../src/Manifests/1.0.1/Roles.php | 381 +
.../src/Manifests/1.0.1/Users.php | 381 +
.../Manifests/swagger-json/applications.json | 1262 ++
.../src/Manifests/swagger-json/management.json | 1905 +++
.../src/Manifests/swagger-json/resources.json | 15 +
.../src/Native/ConfigRepository.php | 102 +
.../src/Native/Facades/Usergrid.php | 118 +
.../src/Native/UsergridBootstrapper.php | 150 +
sdks/php5/apache-usergrid/src/config/config.php | 53 +
.../tests/Api/ApplicationTest.php | 72 +
.../apache-usergrid/tests/Api/AttributeTest.php | 59 +
.../Api/Exception/BadRequestExceptionTest.php | 59 +
.../Api/Exception/InvalidIdExceptionTest.php | 58 +
.../Api/Exception/NotFoundExceptionTest.php | 57 +
.../Api/Exception/ServerErrorExceptionTest.php | 58 +
.../Api/Exception/UnauthorizedExceptionTest.php | 58 +
.../tests/Api/Filters/BooleanTest.php | 45 +
.../tests/Api/Filters/DateTest.php | 41 +
.../tests/Api/ManagementTest.php | 59 +
.../tests/Api/QueryAggregatorTest.php | 40 +
.../apache-usergrid/tests/Api/UsergridTest.php | 153 +
.../tests/Laravel/Facades/UsergridTest.php | 53 +
sdks/php5/apache-usergrid/tests/bootstrap.php | 61 +
stack/build-tools/pom.xml | 5 +-
stack/config/pom.xml | 1 +
.../main/resources/usergrid-default.properties | 37 +-
stack/core/pom.xml | 1 +
.../apache/usergrid/batch/job/package-info.java | 17 -
.../usergrid/batch/repository/package-info.java | 17 -
.../usergrid/batch/service/package-info.java | 17 -
.../org/apache/usergrid/count/package-info.java | 18 -
.../java/org/apache/usergrid/utils/BCrypt.java | 627 -
.../org/apache/usergrid/utils/package-info.java | 17 -
.../main/resources/usergrid-core-context.xml | 41 +-
.../apache/usergrid/batch/job/package-info.java | 17 -
.../org/apache/usergrid/utils/package-info.java | 17 -
.../collection/util/EntityHelper.java | 2 +-
.../persistence/index/CandidateResults.java | 9 +-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 17 +-
stack/launcher/pom.xml | 8 +
.../java/org/apache/usergrid/launcher/App.java | 7 +-
.../launcher/ClasspathStaticHttpHandler.java | 321 -
.../org/apache/usergrid/launcher/Server.java | 5 +-
.../apache/usergrid/launcher/package-info.java | 17 -
stack/mongo-emulator/pom.xml | 3 +-
stack/pom.xml | 10 +-
stack/rest/pom.xml | 1 -
.../rest/applications/ServiceResource.java | 13 +-
.../rest/management/users/UsersResource.java | 14 +-
.../applications/assets/AssetResourceIT.java | 106 +
stack/services/pom.xml | 6 +
.../management/AccountCreationProps.java | 11 +
.../usergrid/management/ManagementService.java | 2 +
.../cassandra/AccountCreationPropsImpl.java | 8 +
.../cassandra/ManagementServiceImpl.java | 30 +-
.../security/crypto/command/BcryptCommand.java | 4 +-
.../security/providers/AbstractProvider.java | 5 +-
.../providers/PingIdentityProvider.java | 19 +-
stack/tools/pom.xml | 1 +
.../org/apache/usergrid/tools/ExportAdmins.java | 236 +
.../org/apache/usergrid/tools/ImportAdmins.java | 451 +
.../apache/usergrid/tools/ResetSuperUser.java | 18 +-
stack/websocket/pom.xml | 1 +
299 files changed, 31666 insertions(+), 91709 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --cc stack/core/src/main/resources/usergrid-core-context.xml
index f33586f,37a2530..c5df523
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@@ -92,17 -96,12 +96,16 @@@
<property orgAppName="maxAttempts" value="10"/>
</bean> -->
-
<bean id="injector"
class="org.apache.usergrid.corepersistence.GuiceFactory">
- <constructor-arg ref="cassandraHostConfigurator" />
- <constructor-arg ref="properties" />
</bean>
+ <!-- <bean id="injector"
+ class="org.apache.usergrid.corepersistence.GuiceFactory">
+ <constructor-arg ref="cassandraHostConfigurator" />
+ <constructor-arg ref="properties" />
+ </bean>-->
+
<bean id="cassandraService"
class="org.apache.usergrid.persistence.cassandra.CassandraService" init-method="init" destroy-method="destroy">
<constructor-arg ref="properties" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/services/pom.xml
----------------------------------------------------------------------
[06/16] incubator-usergrid git commit: Merge remote-tracking branch
'origin/two-dot-o-dev' into USERGRID-541
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o-dev' into USERGRID-541
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/48d8060e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/48d8060e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/48d8060e
Branch: refs/heads/USERGRID-578
Commit: 48d8060e8642e7fb5333f6d65b2ba90b1a5ec292
Parents: 745f355 8f592dd
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 15:31:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 15:31:25 2015 -0600
----------------------------------------------------------------------
portal/index-template.html | 2 +-
portal/js/data/data-controller.js | 58 +++++++++-
portal/js/data/data.html | 36 +++++-
portal/js/global/ug-service.js | 30 ++++-
portal/js/login/login-controller.js | 15 ++-
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../persistence/EntityManagerFactory.java | 6 +-
.../DuplicateUniquePropertyExistsException.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 2 +-
.../persistence/index/IndexRefreshCommand.java | 23 +++-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../index/impl/IndexRefreshCommandImpl.java | 24 ++--
.../src/test/resources/dynamic-test.properties | 2 +-
stack/pom.xml | 12 +-
stack/rest/pom.xml | 16 ++-
.../org/apache/usergrid/rest/IndexResource.java | 4 +-
.../org/apache/usergrid/rest/RootResource.java | 11 +-
.../rest/test/resource2point0/README.md | 111 +++++++++++++++++++
.../resource2point0/endpoints/RootResource.java | 1 +
.../security/providers/AbstractProvider.java | 6 +
.../providers/PingIdentityProvider.java | 14 ++-
.../services/AbstractCollectionService.java | 2 +-
23 files changed, 333 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 961ddd2,c80f80d..fcfba15
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -320,10 -346,10 +320,10 @@@ public class EsEntityIndexImpl implemen
- public Observable<Boolean> refreshAsync() {
+ public Observable<IndexRefreshCommand.IndexRefreshCommandInfo> refreshAsync() {
refreshIndexMeter.mark();
- BetterFuture future = indexBatchBufferProducer.put(new IndexIdentifierImpl.IndexOperationMessage());
+ BetterFuture future = producer.put(new IndexOperationMessage());
future.get();
return indexRefreshCommand.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index 5052ddf,804de51..07150bb
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@@ -143,22 -145,24 +145,24 @@@ public class IndexRefreshCommandImpl im
return future.doOnNext( found -> {
- if ( !found ) {
- logger.error( "Couldn't find record during refresh uuid" + uuid );
+ if ( !found.hasFinished() ) {
+ logger.error(String.format("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ }else{
+ logger.info(String.format("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
}
- } ).doOnCompleted( () -> {
+ } ).doOnCompleted(() -> {
//clean up our data
- String[] aliases = indexCache.getIndexes( alias, AliasedEntityIndex.AliasType.Read );
+ String[] aliases = indexCache.getIndexes(alias, AliasedEntityIndex.AliasType.Read);
DeIndexRequest deIndexRequest =
- new DeIndexRequest( aliases, appScope, edge, entity.getId(), entity.getVersion() );
+ new DeIndexRequest(aliases, appScope, edge, entity.getId(), entity.getVersion());
//delete the item
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage =
- new IndexIdentifierImpl.IndexOperationMessage();
- indexOperationMessage.addDeIndexRequest(deIndexRequest);
- producer.put(indexOperationMessage);
+ IndexOperationMessage indexOperationMessage =
+ new IndexOperationMessage();
+ indexOperationMessage.addDeIndexRequest( deIndexRequest );
+ producer.put( indexOperationMessage );
refreshTimer.stop();
- } );
+ });
}
}
[04/16] incubator-usergrid git commit: Updated wiring to allow spring
beans to be used in guice
Posted by sf...@apache.org.
Updated wiring to allow spring beans to be used in guice
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6f87fff5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6f87fff5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6f87fff5
Branch: refs/heads/USERGRID-578
Commit: 6f87fff520fa14341fef74c8e85fc8456ddc02a8
Parents: 84d779f
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 11:57:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 11:58:40 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../corepersistence/CpEntityManager.java | 19 +--
.../corepersistence/CpEntityManagerFactory.java | 15 ++-
.../corepersistence/CpRelationManager.java | 12 +-
.../usergrid/corepersistence/GuiceFactory.java | 66 +++++-----
.../usergrid/persistence/PersistenceModule.java | 124 +++----------------
.../main/resources/usergrid-core-context.xml | 8 +-
.../resources/usergrid-custom-test.properties | 2 -
8 files changed, 89 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index ffc7628..c21a1ff 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -82,10 +82,9 @@ cassandra.cluster=Test Cluster
cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
#cassandra.keyspace.strategy=org.apache.cassandra.locator.NetworkTopologyStrategy
-#cassandra.keyspace.strategy.options.replication_factor=1
-#cassandra.keyspace.strategy.options.us-east=1
cassandra.keyspace.replication=replication_factor:1
+#cassandra.keyspace.replication=us-east:3
cassandra.username=
cassandra.password=
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a615a43..1506551 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -165,17 +166,19 @@ public class CpEntityManager implements EntityManager {
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
- private UUID applicationId;
+ private final UUID applicationId;
private Application application;
- private ManagerCache managerCache;
+ private final ManagerCache managerCache;
- private ApplicationScope applicationScope;
+ private final ApplicationScope applicationScope;
- private CassandraService cass;
+ private final CassandraService cass;
- private CounterUtils counterUtils;
+ private final CounterUtils counterUtils;
+
+ private final IndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -215,16 +218,18 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final IndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
+ Preconditions.checkNotNull( indexService, "indexService must not be null" );
this.managerCache = managerCache;
this.applicationId = applicationId;
+ this.indexService = indexService;
applicationScope = CpNamingUtils.getApplicationScope( applicationId );
@@ -734,7 +739,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, this, applicationId, entityRef );
+ new CpRelationManager( metricsFactory, managerCache, indexService, this, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e03ed47..6ad8616 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -39,6 +39,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
+
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.*;
@@ -109,9 +111,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
+ private final IndexService indexService;
- public CpEntityManagerFactory(
- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) {
+ public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
+ final Injector injector) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@ -120,7 +123,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(getManagementEntityManager());
+ this.indexService = injector.getInstance( IndexService.class );
+ this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
+ getManagementEntityManager() );
+
}
@@ -175,8 +181,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
-
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, managerCache, metricsFactory, applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 3c72b60..a0a44b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
@@ -117,18 +118,23 @@ public class CpRelationManager implements RelationManager {
private org.apache.usergrid.persistence.model.entity.Entity cpHeadEntity;
- private ApplicationScope applicationScope;
+ private final ApplicationScope applicationScope;
+
+ private final IndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager(final MetricsFactory metricsFactory, final ManagerCache managerCache, final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final IndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+
Assert.notNull( em, "Entity manager cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
Assert.notNull( headEntity, "Head entity cannot be null" );
Assert.notNull( headEntity.getUuid(), "Head entity uuid cannot be null" );
+ Assert.notNull( indexService, "indexService cannot be null" );
+
// TODO: this assert should not be failing
//Assert.notNull( indexBucketLocator, "indexBucketLocator cannot be null" );
this.em = em;
@@ -155,6 +161,8 @@ public class CpRelationManager implements RelationManager {
Assert.notNull( cpHeadEntity, String
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
+ this.indexService = indexService;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
index 3a08034..6a87005 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
@@ -22,20 +22,22 @@ package org.apache.usergrid.corepersistence;
import java.util.Properties;
+import javax.inject.Named;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.persistence.PersistenceModule;
+import com.google.common.base.Preconditions;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.google.inject.Singleton;
import com.netflix.config.ConfigurationManager;
import me.prettyprint.cassandra.service.CassandraHost;
@@ -45,25 +47,28 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator;
/**
* Factory for configuring Guice then returning it
*/
-@Singleton
+@Component
public class GuiceFactory implements FactoryBean<Injector> {
private static final Logger logger = LoggerFactory.getLogger( GuiceFactory.class );
- private final CassandraHostConfigurator chc;
+ @Autowired
+ private CassandraHostConfigurator chc;
- private final Properties systemProperties;
+ @Autowired
+ @Named("properties")
+ private Properties systemProperties;
+ @Autowired
private ApplicationContext applicationContext;
private Injector injector;
-
- public GuiceFactory( final ApplicationContext applicationContext, final CassandraHostConfigurator chc, final Properties systemProperties ) {
- this.chc = chc;
- this.systemProperties = systemProperties;
- this.applicationContext = applicationContext;
+ public GuiceFactory() {
+ // this.chc = chc;
+ // this.systemProperties = systemProperties;
+ // this.applicationContext = applicationContext;
}
@@ -83,15 +88,16 @@ public class GuiceFactory implements FactoryBean<Injector> {
if ( hosts.length == 0 ) {
throw new RuntimeException( "Fatal error: no Cassandra hosts configured" );
}
- String sep = "";
+
for ( CassandraHost host : hosts ) {
if ( StringUtils.isEmpty( host.getHost() ) ) {
throw new RuntimeException( "Fatal error: Cassandra hostname cannot be empty" );
}
- hostsString = hostsString + sep + host.getHost();
- sep = ",";
+ hostsString += host.getHost() + ",";
}
+ hostsString = hostsString.substring( 0, hostsString.length() - 1 );
+
logger.info( "hostsString: " + hostsString );
Properties cpProps = new Properties();
@@ -99,21 +105,14 @@ public class GuiceFactory implements FactoryBean<Injector> {
// Some Usergrid properties must be mapped to Core Persistence properties
cpProps.put( "cassandra.hosts", hostsString );
cpProps.put( "cassandra.port", hosts[0].getPort() );
- cpProps.put( "cassandra.cluster_name", systemProperties.getProperty( "cassandra.cluster" ) );
- String cassRemoteString = ( String ) systemProperties.getProperty( "cassandra.use_remote" );
- if ( cassRemoteString != null && cassRemoteString.equals( "false" ) ) {
- cpProps.put( "cassandra.embedded", "true" );
- }
- else {
- cpProps.put( "cassandra.embedded", "false" );
- }
+ cpProps.put( "cassandra.cluster_name", getAndValidateProperty( "cassandra.cluster" ) );
- cpProps.put( "collections.keyspace.strategy.class",
- systemProperties.getProperty( "cassandra.keyspace.strategy" ) );
+ cpProps
+ .put( "collections.keyspace.strategy.class", getAndValidateProperty( "cassandra.keyspace.strategy" ) );
cpProps.put( "collections.keyspace.strategy.options",
- systemProperties.getProperty( "cassandra.keyspace.replication" ) );
+ getAndValidateProperty( "cassandra.keyspace.replication" ) );
logger.debug( "Set Cassandra properties for Core Persistence: " + cpProps.toString() );
@@ -128,14 +127,23 @@ public class GuiceFactory implements FactoryBean<Injector> {
}
-
- //this is seriously fugly, and needs removed we shouldn't be mixing spring and guice
- injector = Guice.createInjector( new CoreModule( ), new PersistenceModule( applicationContext ) );
+ //we have to inject a couple of spring beans into our Guice. Wire it with PersistenceModule
+ injector = Guice.createInjector( new CoreModule(), new PersistenceModule( applicationContext ) );
return injector;
}
+ private String getAndValidateProperty( final String propName ) {
+
+ final String propValue = systemProperties.getProperty( propName );
+
+ Preconditions.checkNotNull( propValue, propName + " cannot be unset. Set this in your properties" );
+
+ return propValue;
+ }
+
+
@Override
public Class<?> getObjectType() {
return Injector.class;
@@ -146,6 +154,4 @@ public class GuiceFactory implements FactoryBean<Injector> {
public boolean isSingleton() {
return true;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
index 6e58676..70fff90 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
@@ -20,32 +20,15 @@
package org.apache.usergrid.persistence;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.Collection;
-import java.util.Properties;
-
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.context.ApplicationContext;
-import org.apache.usergrid.locking.cassandra.HectorLockManagerImpl;
-import com.google.common.base.Preconditions;
-import com.google.common.io.CharSource;
-import com.google.common.io.Resources;
import com.google.inject.AbstractModule;
-import com.google.inject.Inject;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-import com.google.inject.name.Names;
+import com.google.inject.Provider;
import com.google.inject.spring.SpringIntegration;
-import me.prettyprint.cassandra.connection.RoundRobinBalancingPolicy;
-import me.prettyprint.cassandra.service.CassandraHostConfigurator;
-import me.prettyprint.cassandra.service.ThriftCluster;
-import me.prettyprint.hector.api.Cluster;
-
/**
* Replacement for configuration of our spring modules with guice
@@ -66,97 +49,18 @@ public class PersistenceModule extends AbstractModule {
@Override
protected void configure() {
- SpringIntegration.bindAll( binder(), applicationContext );
- }
+ //bind the application context to our guice instance
+
+ final BeanFactory beanFactory = applicationContext.getAutowireCapableBeanFactory();
+ bind( BeanFactory.class).toInstance( beanFactory);
+
+ //create our bridge and put the EMF into guice. Ultimately all spring needs removed and we need to just use guice
+
+ final Provider<EntityManagerFactory> emfProvider = SpringIntegration.fromSpring( EntityManagerFactory.class, "entityManagerFactory" );
+
+ bind( EntityManagerFactory.class ).toProvider( emfProvider );
+ }
-// <bean id="cassandraCluster" class="me.prettyprint.cassandra.service.ThriftCluster">
-// <constructor-arg value="${cassandra.cluster}" />
-// <constructor-arg ref="cassandraHostConfigurator" />
-// </bean>
-// @Provides
-// @Singleton
-// @Inject
-// public Cluster configureThrift( @Named( "cassandra.cluster" ) final String cassCluster,
-// @Named( "cassandra.connections" ) final int cassandraConnections ){
-//
-// final int setSize = cassandraConnections == 0 ? 50: cassandraConnections;
-//
-// CassandraHostConfigurator hostConfigurator = new CassandraHostConfigurator( cassCluster );
-//
-// hostConfigurator.setMaxActive( setSize );
-// hostConfigurator.setLoadBalancingPolicy( new RoundRobinBalancingPolicy() );
-//
-//
-// ThriftCluster thriftCluster = new ThriftCluster(cassCluster, hostConfigurator);
-//
-// return thriftCluster;
-//
-// }
-//
-//
-// @Provides
-// @Singleton
-// @Inject
-// public Properties configureProps(final PropertiesProvider propertiesProvider ){
-//
-// final Properties props = new Properties( );
-//
-// for(final String propFile: propertiesProvider.getPropertiesFiles()){
-//
-// final URL url = Resources.getResource( propFile );
-//
-// Preconditions.checkNotNull( url, "Could not find properties file '" + propFile + "' on the classpath" );
-//
-//
-// final CharSource propsInput = Resources.asCharSource( url, Charset.defaultCharset() );
-// try {
-// props.load( propsInput.openStream() );
-// }
-// catch ( IOException e ) {
-// throw new RuntimeException( "Unable to load properties file '" + propFile + "'", e );
-// }
-// }
-//
-// //bind these properties
-// Names.bindProperties( binder(), props );
-//
-// return props;
-// }
-//
-// @Provides
-// @Singleton
-// @Inject
-// public void configureLocks(final Cluster hectorCluster, @Named("cassandra.lock.keyspace") final String lockKeyspace, @Named("cassandra.lock.keyspace") final String writeCl, final String readCl ){
-//
-//
-// final HectorLockManagerImpl hectorLockManager = new HectorLockManagerImpl();
-//
-//
-////
-//// <bean name="consistencyLevelPolicy" class="me.prettyprint.cassandra.model.ConfigurableConsistencyLevel">
-//// <property name="defaultReadConsistencyLevel" value="${cassandra.readcl}"/>
-//// <property name="defaultWriteConsistencyLevel" value="${cassandra.writecl}"/>
-//// </bean>
-//
-//// <bean name="lockManager" class="org.apache.usergrid.locking.cassandra.HectorLockManagerImpl" >
-//// <property name="cluster" ref="cassandraCluster"/>
-//// <property name="keyspaceName" value="${cassandra.lock.keyspace}"/>
-//// <property name="consistencyLevelPolicy" ref="consistencyLevelPolicy"/>
-//// </bean>
-//
-// }
-//
-//
-// /**
-// * Interface to allow users to provide and inject properties
-// */
-// public interface PropertiesProvider{
-// /**
-// * Get the properties files to load
-// * @return
-// */
-// String[] getPropertiesFiles();
-// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index 0be6d00..f33586f 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -95,10 +95,14 @@
<bean id="injector"
class="org.apache.usergrid.corepersistence.GuiceFactory">
- <constructor-arg ref="cassandraHostConfigurator" />
- <constructor-arg ref="properties" />
</bean>
+ <!-- <bean id="injector"
+ class="org.apache.usergrid.corepersistence.GuiceFactory">
+ <constructor-arg ref="cassandraHostConfigurator" />
+ <constructor-arg ref="properties" />
+ </bean>-->
+
<bean id="cassandraService"
class="org.apache.usergrid.persistence.cassandra.CassandraService" init-method="init" destroy-method="destroy">
<constructor-arg ref="properties" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 93ef28b..703424f 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -14,8 +14,6 @@
# these settings allow tests to run and consistently pass on 16GB MacBook Pro
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
-cassandra.startup=external
-elasticsearch.startup=external
cassandra.timeout=2000
cassandra.connections=1000
[13/16] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into tempbranch
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' into tempbranch
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1c207d76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1c207d76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1c207d76
Branch: refs/heads/USERGRID-578
Commit: 1c207d76e29980b217613231195e58905da5af38
Parents: 5945574 4cc2e0b
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 18:28:01 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 18:28:01 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 +++++--
.../main/resources/usergrid-core-context.xml | 20 +-------------------
.../usergrid/persistence/EntityManagerIT.java | 8 ++------
.../usergrid/persistence/index/IndexFig.java | 15 +++++++++++++++
.../impl/EsApplicationEntityIndexImpl.java | 4 +++-
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
6 files changed, 27 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
[02/16] incubator-usergrid git commit: Initial pass at moving queues
to core
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
new file mode 100644
index 0000000..8bda3b0
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence;
+
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.support.DefaultListableBeanFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import com.google.inject.Injector;
+
+
+/**
+ * This allows us to search guice for beans that are not in spring
+ */
+@Component
+public class GuiceAdapterBeanFactory extends DefaultListableBeanFactory {
+
+ /**
+ * Wire our injector into this so we can use it go get beans
+ */
+ @Autowired
+ private Injector injector;
+
+
+ public <T> T getBean( Class<T> requiredType ) throws BeansException {
+ final T bean = super.getBean( requiredType );
+
+ // Comes from spring, return it
+ if ( bean != null ) {
+ return bean;
+ }
+
+ final T guiceBean = injector.getInstance( requiredType );
+
+ if(guiceBean == null){
+ throw new NoGuiceBean( "Could not find bean for class" + requiredType );
+ }
+
+
+ return guiceBean;
+ }
+
+
+ @Override
+ public Object getBean( final String name ) throws BeansException {
+ final Object springBean = super.getBean( name );
+
+ return validateBean( springBean, name );
+ }
+
+
+ @Override
+ public <T> T getBean( final String name, final Class<T> requiredType ) throws BeansException {
+ final T springBean = super.getBean( name, requiredType );
+
+ return validateBean( springBean, name );
+ }
+
+
+ @Override
+ public <T> T getBean( final String name, final Class<T> requiredType, final Object... args ) throws BeansException {
+ final T springBean = super.getBean( name, requiredType, args );
+
+ return validateBean( springBean, name );
+ }
+
+
+ /**
+ * If we can't find the spring bean, we should blow up
+ * @param springBean
+ * @param <T>
+ * @return
+ */
+ private <T> T validateBean( T springBean, final String name ) {
+ if ( springBean == null ) {
+ throw new NoGuiceBean( String.format("Guice beans by name is unsupoported, and could not find a spring bean with the name '%s'", name) );
+ }
+
+ return springBean;
+ }
+
+
+ /**
+ * Exception class to throw when we can't find a bean in spring, and can't find it in guice
+ */
+ public static final class NoGuiceBean extends BeansException{
+
+ public NoGuiceBean( final String msg ) {
+ super( msg );
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
new file mode 100644
index 0000000..6e58676
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence;
+
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.Properties;
+
+import org.springframework.context.ApplicationContext;
+
+import org.apache.usergrid.locking.cassandra.HectorLockManagerImpl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.CharSource;
+import com.google.common.io.Resources;
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
+import com.google.inject.spring.SpringIntegration;
+
+import me.prettyprint.cassandra.connection.RoundRobinBalancingPolicy;
+import me.prettyprint.cassandra.service.CassandraHostConfigurator;
+import me.prettyprint.cassandra.service.ThriftCluster;
+import me.prettyprint.hector.api.Cluster;
+
+
+/**
+ * Replacement for configuration of our spring modules with guice
+ */
+public class PersistenceModule extends AbstractModule {
+
+
+
+ private final ApplicationContext applicationContext;
+
+ public PersistenceModule( final ApplicationContext applicationContext ) {
+
+ this.applicationContext = applicationContext;
+ }
+
+
+
+
+ @Override
+ protected void configure() {
+ SpringIntegration.bindAll( binder(), applicationContext );
+ }
+
+
+
+// <bean id="cassandraCluster" class="me.prettyprint.cassandra.service.ThriftCluster">
+// <constructor-arg value="${cassandra.cluster}" />
+// <constructor-arg ref="cassandraHostConfigurator" />
+// </bean>
+// @Provides
+// @Singleton
+// @Inject
+// public Cluster configureThrift( @Named( "cassandra.cluster" ) final String cassCluster,
+// @Named( "cassandra.connections" ) final int cassandraConnections ){
+//
+// final int setSize = cassandraConnections == 0 ? 50: cassandraConnections;
+//
+// CassandraHostConfigurator hostConfigurator = new CassandraHostConfigurator( cassCluster );
+//
+// hostConfigurator.setMaxActive( setSize );
+// hostConfigurator.setLoadBalancingPolicy( new RoundRobinBalancingPolicy() );
+//
+//
+// ThriftCluster thriftCluster = new ThriftCluster(cassCluster, hostConfigurator);
+//
+// return thriftCluster;
+//
+// }
+//
+//
+// @Provides
+// @Singleton
+// @Inject
+// public Properties configureProps(final PropertiesProvider propertiesProvider ){
+//
+// final Properties props = new Properties( );
+//
+// for(final String propFile: propertiesProvider.getPropertiesFiles()){
+//
+// final URL url = Resources.getResource( propFile );
+//
+// Preconditions.checkNotNull( url, "Could not find properties file '" + propFile + "' on the classpath" );
+//
+//
+// final CharSource propsInput = Resources.asCharSource( url, Charset.defaultCharset() );
+// try {
+// props.load( propsInput.openStream() );
+// }
+// catch ( IOException e ) {
+// throw new RuntimeException( "Unable to load properties file '" + propFile + "'", e );
+// }
+// }
+//
+// //bind these properties
+// Names.bindProperties( binder(), props );
+//
+// return props;
+// }
+//
+// @Provides
+// @Singleton
+// @Inject
+// public void configureLocks(final Cluster hectorCluster, @Named("cassandra.lock.keyspace") final String lockKeyspace, @Named("cassandra.lock.keyspace") final String writeCl, final String readCl ){
+//
+//
+// final HectorLockManagerImpl hectorLockManager = new HectorLockManagerImpl();
+//
+//
+////
+//// <bean name="consistencyLevelPolicy" class="me.prettyprint.cassandra.model.ConfigurableConsistencyLevel">
+//// <property name="defaultReadConsistencyLevel" value="${cassandra.readcl}"/>
+//// <property name="defaultWriteConsistencyLevel" value="${cassandra.writecl}"/>
+//// </bean>
+//
+//// <bean name="lockManager" class="org.apache.usergrid.locking.cassandra.HectorLockManagerImpl" >
+//// <property name="cluster" ref="cassandraCluster"/>
+//// <property name="keyspaceName" value="${cassandra.lock.keyspace}"/>
+//// <property name="consistencyLevelPolicy" ref="consistencyLevelPolicy"/>
+//// </bean>
+//
+// }
+//
+//
+// /**
+// * Interface to allow users to provide and inject properties
+// */
+// public interface PropertiesProvider{
+// /**
+// * Get the properties files to load
+// * @return
+// */
+// String[] getPropertiesFiles();
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
new file mode 100644
index 0000000..e5254d1
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence;
+
+
+import org.apache.usergrid.persistence.core.guice.TestModule;
+
+
+public class TestIndexModule extends TestModule {
+
+ @Override
+ protected void configure() {
+
+ //this will break, we need to untagle this and move to guice in core completely
+ install( new CoreModule( ));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
new file mode 100644
index 0000000..a76a589
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+@NotThreadSafe
+public class BufferQueueSQSImplTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+ @Inject
+ public QueueManagerFactory queueManagerFactory;
+
+ @Inject
+ public QueryFig queryFig;
+
+ @Inject
+ public MapManagerFactory mapManagerFactory;
+
+ @Inject
+ public MetricsFactory metricsFactory;
+
+
+ private BufferQueueSQSImpl bufferQueueSQS;
+
+ @Before
+ public void setup(){
+ bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
+ }
+
+
+
+
+ @Test
+ public void testMessageIndexing(){
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+ assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
+ assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
+
+ final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
+ final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
+
+
+ final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
+ final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
+
+
+ //de-index request
+ final DeIndexRequest
+ deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
+
+
+ SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
+
+ final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
+ SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
+
+
+
+
+ IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
+ indexOperationMessage.addIndexRequest( indexRequest1);
+ indexOperationMessage.addIndexRequest( indexRequest2);
+
+ indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
+ indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
+
+ bufferQueueSQS.offer( indexOperationMessage );
+
+ //wait for it to send to SQS
+ indexOperationMessage.getFuture().get();
+
+ //now get it back
+
+ final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
+
+ assertTrue(ops.size() > 0);
+
+ final IndexOperationMessage returnedOperation = ops.get( 0 );
+
+ //get the operations out
+
+ final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
+
+ assertTrue(indexRequestSet.contains(indexRequest1));
+ assertTrue(indexRequestSet.contains(indexRequest2));
+
+
+ final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
+
+ assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
+ assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
+
+
+
+ //now ack the message
+
+ bufferQueueSQS.ack( ops );
+
+ }
+
+ private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
+ final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+
+ List<IndexOperationMessage> ops;
+
+ do{
+ ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+ }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
+
+ return ops;
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
new file mode 100644
index 0000000..1b0f538
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.aws;
+
+
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.amazonaws.AmazonClientException;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule implements TestRule {
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+ try {
+ base.evaluate();
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ if ( t instanceof AmazonClientException ) {
+
+ final AmazonClientException ace = ( AmazonClientException ) t;
+
+ if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+ "could not get aws secret key from system properties" ) ) {
+ //swallow
+ return true;
+ }
+ }
+
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 2206953..fa73991 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -28,6 +28,29 @@ import rx.Observable;
* Get all edges from source
*/
public interface EdgesObservable {
+
+ /**
+ * Return an observable of all edges from a source
+ * @param gm
+ * @param sourceNode
+ * @return
+ */
Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode);
+
+ /**
+ * Get all edges from the source node with the target type
+ * @param gm
+ * @param sourceNode
+ * @param targetType
+ * @return
+ */
+ Observable<Edge> getEdgesFromSource(final GraphManager gm, final Id sourceNode, final String targetType );
+
+ /**
+ * Return an observable of all edges to a target
+ * @param gm
+ * @param targetNode
+ * @return
+ */
Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 371cf1d..2264cbd 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -19,62 +19,88 @@
*/
package org.apache.usergrid.persistence.graph.serialization.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
import rx.Observable;
import rx.functions.Func1;
+
/**
* Emits the edges that are edges from the specified source node
*/
public class EdgesObservableImpl implements EdgesObservable {
- private static final Logger logger = LoggerFactory.getLogger(EdgesObservableImpl.class);
- public EdgesObservableImpl(){
+ private static final Logger logger = LoggerFactory.getLogger( EdgesObservableImpl.class );
+
+
+ public EdgesObservableImpl() {
}
+
/**
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode){
- Observable<String> edgeTypes = gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+ public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode ) {
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
- return edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
+ return edgeTypes.flatMap( edgeType -> {
logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
- return gm.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
- }
+ return gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ null ) );
} );
}
+
+
+ @Override
+ public Observable<Edge> getEdgesFromSource( final GraphManager gm, final Id sourceNode, final String targetType ) {
+
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+
+
+ return edgeTypes.flatMap( edgeType -> {
+
+ logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
+
+ return gm.loadEdgesFromSourceByType(
+ new SimpleSearchByIdType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ targetType, null ) );
+ } );
+ }
+
+
/**
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode) {
- Observable<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
+ public Observable<Edge> edgesToTarget( final GraphManager gm, final Id targetNode ) {
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
- return edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
+ return edgeTypes.flatMap( edgeType -> {
- logger.debug( "Loading edges of edgeType {} to {}", edgeType, targetNode);
+ logger.debug( "Loading edges of edgeType {} to {}", edgeType, targetNode );
- return gm.loadEdgesToTarget( new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
- }
+ return gm.loadEdgesToTarget(
+ new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ null ) );
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index da05d39..39c686c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index;/*
import java.util.UUID;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -65,7 +66,7 @@ public interface EntityIndexBatch {
* Execute the batch
* @return future to guarantee execution
*/
- BetterFuture execute();
+ BetterFuture<IndexOperationMessage> execute();
/**
* Get the number of operations in the batch
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 8b9a03e..d13d055 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -30,84 +30,52 @@ import org.apache.usergrid.persistence.index.impl.EsProvider;
@FigSingleton
public interface IndexFig extends GuicyFig {
- public static final String ELASTICSEARCH_HOSTS = "elasticsearch.hosts";
+ String ELASTICSEARCH_HOSTS = "elasticsearch.hosts";
- public static final String ELASTICSEARCH_PORT = "elasticsearch.port";
+ String ELASTICSEARCH_PORT = "elasticsearch.port";
- public static final String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster_name";
+ String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster_name";
- public static final String ELASTICSEARCH_NODENAME = "elasticsearch.node_name";
+ String ELASTICSEARCH_NODENAME = "elasticsearch.node_name";
- public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
+ String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
- public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch.alias_postfix";
+ String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch.alias_postfix";
- public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
+ String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
- public static final String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
+ String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
- public static final String ELASTICSEARCH_NUMBER_OF_REPLICAS = "elasticsearch.number_replicas";
+ String ELASTICSEARCH_NUMBER_OF_REPLICAS = "elasticsearch.number_replicas";
- public static final String QUERY_CURSOR_TIMEOUT_MINUTES = "elasticsearch.cursor_timeout.minutes";
+ String QUERY_CURSOR_TIMEOUT_MINUTES = "elasticsearch.cursor_timeout.minutes";
- public static final String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
+ String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
- public static final String INDEX_BUFFER_SIZE = "elasticsearch.buffer_size";
+ String INDEX_BUFFER_SIZE = "elasticsearch.buffer_size";
- public static final String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
+ String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
- public static final String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
+ String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
- /**
- * Amount of time to wait when reading from the queue
- */
- public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
-
- /**
- * Amount of time to wait when reading from the queue in milliseconds
- */
- public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
-
- public static final String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
-
- public static final String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
+ String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
/**
* the number of times we can fail before we refresh the client
*/
- public static final String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
-
- /**
- * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
- * backpressure
- */
- public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+ String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
- /**
- * The number of worker threads to consume from the queue
- */
- public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
-
-
- public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
/**
* The client type to use. Valid values are NODE or TRANSPORT
*/
- public static final String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+ String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
@Default( "127.0.0.1" )
@@ -145,20 +113,20 @@ public interface IndexFig extends GuicyFig {
@Default( "false" )
@Key( ELASTICSEARCH_FORCE_REFRESH )
- public boolean isForcedRefresh();
+ boolean isForcedRefresh();
/** Identify the client node with a unique name. */
@Default( "default" )
@Key( ELASTICSEARCH_NODENAME )
- public String getNodeName();
+ String getNodeName();
@Default( "6" )
@Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
- public int getNumberOfShards();
+ int getNumberOfShards();
@Default( "1" )
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
- public int getNumberOfReplicas();
+ int getNumberOfReplicas();
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
@@ -181,12 +149,7 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_BUFFER_SIZE )
int getIndexBufferSize();
- /**
- * size of the buffer to build up before you send results
- */
- @Default( "1000" )
- @Key( INDEX_QUEUE_SIZE )
- int getIndexQueueSize();
+
/**
* Request batch size for ES
@@ -199,27 +162,6 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_WRITE_CONSISTENCY_LEVEL )
String getWriteConsistencyLevel();
- @Default( "1000" )
- @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
- long getFailureRetryTime();
-
- //give us 60 seconds to process the message
- @Default( "60" )
- @Key( INDEX_QUEUE_READ_TIMEOUT )
- int getIndexQueueTimeout();
-
- @Default( "2" )
- @Key( ELASTICSEARCH_WORKER_COUNT )
- int getWorkerCount();
-
- @Default( "LOCAL" )
- @Key( ELASTICSEARCH_QUEUE_IMPL )
- String getQueueImplementation();
-
- @Default( "1000" )
- @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
- long getQueueOfferTimeout();
-
/**
* Return the type of client. Valid values or NODE or TRANSPORT
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 10cbde0..b6a1f09 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -52,15 +52,12 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndex.class).to(EsEntityIndexImpl.class).asEagerSingleton();
bind(IndexCache.class).to(EsIndexCacheImpl.class);
bind(IndexRefreshCommand.class).to(IndexRefreshCommandImpl.class);
- bind(FailureMonitorImpl.IndexIdentifier.class).to(IndexIdentifierImpl.class);
+ bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
- bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
- bind( BufferQueue.class).toProvider( QueueProvider.class );
-
//wire up the edg migration. A no-op ATM, but retained for future development
Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder =
Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
deleted file mode 100644
index ea3e046..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.guice;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-
-
-/**
- * A provider to allow users to configure their queue impl via properties
- */
-@Singleton
-public class QueueProvider implements Provider<BufferQueue> {
-
- private final IndexFig indexFig;
-
- private final QueueManagerFactory queueManagerFactory;
- private final MapManagerFactory mapManagerFactory;
- private final MetricsFactory metricsFactory;
-
- private BufferQueue bufferQueue;
-
-
- @Inject
- public QueueProvider( final IndexFig indexFig, final QueueManagerFactory queueManagerFactory,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- this.indexFig = indexFig;
-
-
- this.queueManagerFactory = queueManagerFactory;
- this.mapManagerFactory = mapManagerFactory;
- this.metricsFactory = metricsFactory;
- }
-
-
- @Override
- @Singleton
- public BufferQueue get() {
- if ( bufferQueue == null ) {
- bufferQueue = getQueue();
- }
-
-
- return bufferQueue;
- }
-
-
- private BufferQueue getQueue() {
- final String value = indexFig.getQueueImplementation();
-
- final Implementations impl = Implementations.valueOf( value );
-
- switch ( impl ) {
- case LOCAL:
- return new BufferQueueInMemoryImpl( indexFig );
- case SQS:
- return new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
- default:
- throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
- }
- }
-
-
- private String getErrorValues() {
- String values = "";
-
- for ( final Implementations impl : Implementations.values() ) {
- values += impl + ", ";
- }
-
- values = values.substring( 0, values.length() - 2 );
-
- return values;
- }
-
-
- /**
- * Different implementations
- */
- public static enum Implementations {
- LOCAL,
- SQS;
-
-
- public String asString() {
- return toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
deleted file mode 100644
index cb89f64..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-
-/**
- * A temporary interface of our buffer Q to decouple of producer and consumer;
- */
-public interface BufferQueue {
-
- /**
- * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
- * Other queues may return the future after ack on the offer. See the implementation documentation for details.
- * @param operation
- */
- public void offer(final IndexIdentifierImpl.IndexOperationMessage operation);
-
-
- /**
- * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
- * May return less than the take size, but will never return null
- *
- * @param takeSize
- * @param timeout
- * @param timeUnit
- * @return A null safe lid
- */
- public List<IndexIdentifierImpl.IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
-
-
- /**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
- * This will set the future as done in in memory operations
- *
- * @param messages
- */
- public void ack(final List<IndexIdentifierImpl.IndexOperationMessage> messages);
-
- /**
- * Mark these message as failed. Set the exception in the future on local operation
- *
- * @param messages
- */
- public void fail(final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
deleted file mode 100644
index bfaed3d..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.index.IndexFig;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class BufferQueueInMemoryImpl implements BufferQueue {
-
-
- private final IndexFig fig;
- private final ArrayBlockingQueue<IndexIdentifierImpl.IndexOperationMessage> messages;
-
-
- @Inject
- public BufferQueueInMemoryImpl( final IndexFig fig ) {
- this.fig = fig;
- messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
- }
-
-
- @Override
- public void offer( final IndexIdentifierImpl.IndexOperationMessage operation ) {
- try {
- messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
- }
- catch ( InterruptedException e ) {
- throw new RuntimeException("Unable to offer message to queue", e);
- }
- }
-
-
- @Override
- public List<IndexIdentifierImpl.IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- final List<IndexIdentifierImpl.IndexOperationMessage> response = new ArrayList<>( takeSize );
- try {
-
-
- messages.drainTo( response, takeSize );
-
- //we got something, go process it
- if ( response.size() > 0 ) {
- return response;
- }
-
-
- final IndexIdentifierImpl.IndexOperationMessage polled = messages.poll( timeout, timeUnit );
-
- if ( polled != null ) {
- response.add( polled );
-
- //try to add more
- messages.drainTo( response, takeSize - 1 );
- }
- }
- catch ( InterruptedException e ) {
- //swallow
- }
-
-
- return response;
- }
-
-
- @Override
- public void ack( final List<IndexIdentifierImpl.IndexOperationMessage> messages ) {
- //if we have a future ack it
- for ( final IndexIdentifierImpl.IndexOperationMessage op : messages ) {
- op.done();
- }
- }
-
-
- @Override
- public void fail( final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t ) {
-
-
- for ( final IndexIdentifierImpl.IndexOperationMessage op : messages ) {
- final BetterFuture<IndexIdentifierImpl.IndexOperationMessage> future = op.getFuture();
-
- if ( future != null ) {
- future.setError( t );
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
deleted file mode 100644
index fee828f..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
- * performing
- */
-@Singleton
-public class BufferQueueSQSImpl implements BufferQueue {
-
- private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final MapManager mapManager;
- private final IndexFig indexFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- final QueueScope queueScope =
- new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.indexFig = indexFig;
-
- final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
-
- this.mapManager = mapManagerFactory.createMapManager( scope );
-
-
- this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- @Override
- public void offer( final IndexIdentifierImpl.IndexOperationMessage operation ) {
-
- //no op
- if(operation.isEmpty()){
- operation.getFuture().done();
- return;
- }
-
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //write to cassandra
- this.mapManager.putString( identifier.toString(), payLoad, TTL );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- operation.done();
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public List<IndexIdentifierImpl.IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
-
- final List<IndexIdentifierImpl.IndexOperationMessage> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if(messages.size() == 0){
- return response;
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
- //look up the values
- final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String key = getMessageKey( message );
-
- //now see if the key was there
- final String payload = storedCommands.get( key );
-
- //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
- // a DLQ
-
- if ( payload == null ) {
- continue;
- }
-
- final IndexIdentifierImpl.IndexOperationMessage messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public void ack( final List<IndexIdentifierImpl.IndexOperationMessage> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexIdentifierImpl.IndexOperationMessage ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
-
- //remove it from the map
- mapManager.delete( key );
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- @Override
- public void fail( final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t ) {
- //no op, just let it retry after the queue timeout
- }
-
-
- /** Read the object from Base64 string. */
- private IndexIdentifierImpl.IndexOperationMessage fromString( String s ) throws IOException {
- IndexIdentifierImpl.IndexOperationMessage o = mapper.readValue( s, IndexIdentifierImpl.IndexOperationMessage.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexIdentifierImpl.IndexOperationMessage o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
- private String getMessageKey(final QueueMessage message){
- return message.getBody().toString();
- }
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexIdentifierImpl.IndexOperationMessage {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexIdentifierImpl.IndexOperationMessage source ) {
- this.message = message;
- this.addAllDeIndexRequest( source.getDeIndexRequests() );
- this.addAllIndexRequest( source.getIndexRequests() );
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index b509ad9..05c84b9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.elasticsearch.action.ActionListener;
@@ -61,14 +60,11 @@ import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import rx.Observable;
@@ -85,12 +81,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final ApplicationScope applicationScope;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
private final Timer searchTimer;
private final Timer cursorTimer;
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -103,11 +99,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
@Inject
public EsApplicationEntityIndexImpl( ApplicationScope appScope, final AliasedEntityIndex entityIndex,
- final IndexFig config, final IndexBufferProducer indexBatchBufferProducer,
+ final IndexFig config, final IndexBufferConsumer indexBatchBufferProducer,
final EsProvider provider,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig,
- final FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
+ final IndexIdentifier indexIdentifier ) {
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.indexFig = indexFig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index d22f000..32a0f02 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -44,18 +44,18 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final ApplicationScope applicationScope;
private final IndexAlias alias;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final AliasedEntityIndex entityIndex;
- private IndexIdentifierImpl.IndexOperationMessage container;
+ private IndexOperationMessage container;
public EsEntityIndexBatchImpl( final ApplicationScope applicationScope,
- final IndexBufferProducer indexBatchBufferProducer,
+ final IndexBufferConsumer indexBatchBufferProducer,
final AliasedEntityIndex entityIndex,
- FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
+ IndexIdentifier indexIdentifier ) {
this.applicationScope = applicationScope;
this.indexBatchBufferProducer = indexBatchBufferProducer;
@@ -63,7 +63,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.indexIdentifier = indexIdentifier;
this.alias = indexIdentifier.getAlias();
//constrained
- this.container = new IndexIdentifierImpl.IndexOperationMessage();
+ this.container = new IndexOperationMessage();
}
@@ -126,8 +126,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public BetterFuture execute() {
- IndexIdentifierImpl.IndexOperationMessage tempContainer = container;
- container = new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage tempContainer = container;
+ container = new IndexOperationMessage();
/**
* No-op, just disregard it
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 6145069..e41bcf8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -42,12 +42,12 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
private final IndexFig indexFig;
private final AliasedEntityIndex entityIndex;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
private LoadingCache<ApplicationScope, ApplicationEntityIndex> eiCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, ApplicationEntityIndex>() {
@@ -60,9 +60,9 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
@Inject
public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider,
- final IndexBufferProducer indexBatchBufferProducer,
+ final IndexBufferConsumer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
- final IndexFig indexFig, final AliasedEntityIndex entityIndex, final FailureMonitorImpl.IndexIdentifier indexIdentifier ){
+ final IndexFig indexFig, final AliasedEntityIndex entityIndex, final IndexIdentifier indexIdentifier ){
this.config = config;
this.provider = provider;
this.indexBatchBufferProducer = indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 456bd15..961ddd2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -25,57 +25,33 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.sun.org.apache.xpath.internal.operations.Bool;
-import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
-import org.apache.usergrid.persistence.index.query.ParsedQuery;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.EntityUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
-import org.elasticsearch.action.index.*;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
-import org.elasticsearch.indices.IndexMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import rx.*;
+
import rx.Observable;
-import rx.functions.Action1;
import java.io.IOException;
import java.net.URL;
import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -87,8 +63,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final IndexAlias alias;
- private final IndexBufferProducer indexBatchBufferProducer;
- private final MetricsFactory metricsFactory;
+ private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer addTimer;
private final Timer updateAliasTimer;
@@ -97,7 +72,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
* We purposefully make this per instance. Some indexes may work, while others may fail
*/
private final EsProvider esProvider;
- private final IndexBufferProducer producer;
private final IndexRefreshCommand indexRefreshCommand;
//number of times to wait for the index to refresh properly.
@@ -110,8 +84,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
ImmutableMap.<String, Object>builder().put(IndexingUtils.ENTITY_ID_FIELDNAME, UUIDGenerator.newTimeUUID().toString()).build();
- private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+
+ private final IndexIdentifier indexIdentifier;
private IndexCache aliasCache;
private Timer mappingTimer;
@@ -121,12 +95,12 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
@Inject
- public EsEntityIndexImpl(final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final IndexCache indexCache, final MetricsFactory metricsFactory,
- final IndexFig indexFig, final FailureMonitorImpl.IndexIdentifier indexIdentifier,
- final IndexBufferProducer producer, IndexRefreshCommand indexRefreshCommand) {
- this.indexBatchBufferProducer = indexBatchBufferProducer;
- this.metricsFactory = metricsFactory;
+ public EsEntityIndexImpl( final EsProvider provider,
+ final IndexCache indexCache,
+ final IndexFig indexFig, final IndexIdentifier indexIdentifier,
+ final IndexBufferConsumer producer, final IndexRefreshCommand indexRefreshCommand,
+ final MetricsFactory metricsFactory) {
+
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
@@ -349,7 +323,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
public Observable<Boolean> refreshAsync() {
refreshIndexMeter.mark();
- BetterFuture future = indexBatchBufferProducer.put(new IndexIdentifierImpl.IndexOperationMessage());
+ BetterFuture future = producer.put(new IndexOperationMessage());
future.get();
return indexRefreshCommand.execute();
}
[14/16] incubator-usergrid git commit: Fixes several wiring bugs
Posted by sf...@apache.org.
Fixes several wiring bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ee1a66fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ee1a66fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ee1a66fa
Branch: refs/heads/USERGRID-578
Commit: ee1a66fabd61f032a8748db2d5914171fd8ae59a
Parents: 1c207d7
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 19:09:46 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 19:09:46 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 ++-
.../corepersistence/index/IndexService.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 13 +++--
.../rx/impl/AllEntityIdsObservableImpl.java | 5 ++
.../corepersistence/index/PublishRxtest.java | 61 ++++++++++++++++++++
.../index/impl/EsEntityIndexImpl.java | 2 +-
6 files changed, 84 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index f49e066..bb936b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -34,6 +34,8 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
@@ -102,7 +104,7 @@ public class CoreModule extends AbstractModule {
@Override
public void configureMigrationProvider() {
bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
- AllApplicationsObservable.class );
+ AllApplicationsObservableImpl.class );
}
});
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
@@ -141,6 +143,7 @@ public class CoreModule extends AbstractModule {
plugins.addBinding().to( MigrationModuleVersionPlugin.class );
bind( AllApplicationsObservable.class ).to( AllApplicationsObservableImpl.class );
+ bind( AllEntityIdsObservable.class).to( AllEntityIdsObservableImpl.class );
/*****
@@ -148,7 +151,7 @@ public class CoreModule extends AbstractModule {
*****/
- bind(IndexService.class).to( IndexServiceImpl.class );
+ bind( IndexService.class ).to( IndexServiceImpl.class );
//bind the queue provider
bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index 2bf073c..41fe323 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -41,9 +41,11 @@ public interface IndexService {
* @param applicationScope The scope of the entity
* @param entity The entity
*
- * @return An observable with the count of every
+ * @return An observable with the count of every batch executed to index the entity. Note that this a cold observable
+ * and must be subscribed to in order to perform the operation. This also makes no assumptions on scheduling. It is up to the caller
+ * to assign the correct scheduler to the observable
*/
- Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity );
+ Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 873e2b6..29dd04a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.Collection;
+
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.graph.Edge;
@@ -40,6 +42,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
+import rx.observables.ConnectableObservable;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeToTarget;
@@ -69,7 +72,7 @@ public class IndexServiceImpl implements IndexService {
@Override
- public Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
+ public Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
//bootstrap the lower modules from their caches
@@ -93,7 +96,8 @@ public class IndexServiceImpl implements IndexService {
final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
- final Observable<IndexOperationMessage> observable =
+ //start the observable via publish
+ final ConnectableObservable<IndexOperationMessage> observable =
//try to send a whole batch if we can
Observable.merge( sourceEdgesToIndex, targetSizes ).buffer( indexFig.getIndexBatchSize() )
@@ -101,12 +105,11 @@ public class IndexServiceImpl implements IndexService {
.flatMap( buffer -> Observable.from( buffer ).collect( () -> ei.createBatch(),
( batch, indexEdge ) -> batch.index( indexEdge, entity ) )
//return the future from the batch execution
- .flatMap( batch -> Observable.from( batch.execute() ) ) );
+ .flatMap( batch -> Observable.from( batch.execute() ) ) ).publish();
- observable.toBlocking().last();
- return Observable.just( 0 );
+ return observable.countLong();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index 1f74420..f9df1f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -21,7 +21,10 @@ package org.apache.usergrid.corepersistence.rx.impl;
+
import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -37,12 +40,14 @@ import rx.Observable;
/**
* An implementation that will provide all entityId scopes in the system
*/
+@Singleton
public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
private final GraphManagerFactory graphManagerFactory;
private final TargetIdObservable targetIdObservable;
private final EdgesObservable edgesObservable;
+ @Inject
public AllEntityIdsObservableImpl( final GraphManagerFactory graphManagerFactory,
final TargetIdObservable targetIdObservable,
final EdgesObservable edgesObservable ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
new file mode 100644
index 0000000..1eb00c2
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import rx.Observable;
+import rx.Subscription;
+import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Validates the Rx scheduler works as expected with publish
+ */
+public class PublishRxTest {
+
+ @Test
+ public void testPublish() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final Subscription connectedObservable =
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ .subscribe();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();;
+
+ assertTrue("Subscription complete", completedSubscription);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fcfba15..dcda045 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -275,7 +275,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
// delete all types, this way if we miss one it will get cleaned up
esProvider.getClient().prepareDelete( alias.getWriteAlias(), VERIFY_TYPE, tempId).get();
- logger.info( "Successfully deleted documents in index {} read {} write {} and type {} with id {}",
+ logger.info( "Successfully deleted documents in read {} write {} and type {} with id {}",
alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE, tempId );
return true;