You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/24 18:00:27 UTC
[01/52] [abbrv] incubator-usergrid git commit: Initial pass at moving
queues to core
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-593 6904c9dbd -> ec3a710ad
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index e55fcc2..6bcc405 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.persistence.index.impl;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -33,19 +32,19 @@ import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexFig;
import com.codahale.metrics.Counter;
-import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
+import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
import rx.Subscriber;
-import rx.Subscription;
import rx.schedulers.Schedulers;
@@ -54,179 +53,110 @@ import rx.schedulers.Schedulers;
*/
@Singleton
public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
- private static final Logger log = LoggerFactory.getLogger(EsIndexBufferConsumerImpl.class);
+ private static final Logger log = LoggerFactory.getLogger( EsIndexBufferConsumerImpl.class );
private final IndexFig config;
private final FailureMonitorImpl failureMonitor;
private final Client client;
private final Timer flushTimer;
- private final Counter indexSizeCounter;
private final Counter indexErrorCounter;
private final Meter flushMeter;
private final Timer produceTimer;
- private final BufferQueue bufferQueue;
private final IndexFig indexFig;
- private final AtomicLong counter = new AtomicLong( );
+ private final AtomicLong counter = new AtomicLong();
+
+
+ private final Counter indexSizeCounter;
- //the actively running subscription
- private List<Subscription> subscriptions;
+ private final Timer offerTimer;
- private Object mutex = new Object();
+ private final BufferProducer bufferProducer;
+
+
+ private AtomicLong inFlight = new AtomicLong();
- private AtomicLong inFlight = new AtomicLong( );
@Inject
- public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider, final MetricsFactory
- metricsFactory, final BufferQueue bufferQueue, final IndexFig indexFig ){
+ public EsIndexBufferConsumerImpl( final IndexFig config, final EsProvider provider,
+ final MetricsFactory metricsFactory, final IndexFig indexFig ) {
- this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "buffer.flush");
- this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "buffer.meter");
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "buffer.size");
- this.indexErrorCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "error.count");
+ this.flushTimer = metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "buffer.flush" );
+ this.flushMeter = metricsFactory.getMeter( EsIndexBufferConsumerImpl.class, "buffer.meter" );
+ this.indexSizeCounter = metricsFactory.getCounter( EsIndexBufferConsumerImpl.class, "buffer.size" );
+ this.indexErrorCounter = metricsFactory.getCounter( EsIndexBufferConsumerImpl.class, "error.count" );
+ this.offerTimer = metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "index.buffer.producer.timer" );
//wire up the gauge of inflight messages
- metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", new Gauge<Long>() {
- @Override
- public Long getValue() {
- return inFlight.longValue();
- }
- } );
-
+ metricsFactory.addGauge( EsIndexBufferConsumerImpl.class, "inflight.meter", () -> inFlight.longValue() );
this.config = config;
- this.failureMonitor = new FailureMonitorImpl(config,provider);
+ this.failureMonitor = new FailureMonitorImpl( config, provider );
this.client = provider.getClient();
- this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
- this.bufferQueue = bufferQueue;
+ this.produceTimer =
+ metricsFactory.getTimer( EsIndexBufferConsumerImpl.class, "index.buffer.consumer.messageFetch" );
this.indexFig = indexFig;
- subscriptions = new ArrayList<>( indexFig.getWorkerCount() );
+ this.bufferProducer = new BufferProducer();
//batch up sets of some size and send them in batch
- start();
- }
+ startSubscription();
+ }
- /**
- * Loop throught and start the workers
- */
- public void start() {
- final int count = indexFig.getWorkerCount();
- for(int i = 0; i < count; i ++){
- startWorker();
- }
+ public BetterFuture put( IndexOperationMessage message ) {
+ Preconditions.checkNotNull( message, "Message cannot be null" );
+ indexSizeCounter.inc( message.getDeIndexRequests().size() );
+ indexSizeCounter.inc( message.getIndexRequests().size() );
+ Timer.Context time = offerTimer.time();
+ bufferProducer.send( message );
+ time.stop();
+ return message.getFuture();
}
/**
- * Stop the workers
+ * Start the subscription
*/
- public void stop() {
- synchronized ( mutex ) {
- //stop consuming
-
- for(final Subscription subscription: subscriptions){
- subscription.unsubscribe();
- }
- }
- }
-
-
- private void startWorker(){
- synchronized ( mutex) {
-
- Observable<List<IndexIdentifierImpl.IndexOperationMessage>> consumer = Observable.create(
- new Observable.OnSubscribe<List<IndexIdentifierImpl.IndexOperationMessage>>() {
- @Override
- public void call( final Subscriber<? super List<IndexIdentifierImpl.IndexOperationMessage>> subscriber ) {
-
- //name our thread so it's easy to see
- Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
-
-
- List<IndexIdentifierImpl.IndexOperationMessage> drainList = null;
-
- do {
-
- Timer.Context timer = produceTimer.time();
+ private void startSubscription() {
- try {
+ Observable.create( bufferProducer )
+ .buffer( indexFig.getIndexBufferSize(), indexFig.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
+ Schedulers.newThread() )
+ .doOnNext( containerList -> {
+ if ( containerList.size() == 0 ) {
+ return;
+ }
- drainList = bufferQueue
- .take( config.getIndexBufferSize(), config.getIndexBufferTimeout(),
- TimeUnit.MILLISECONDS );
+ flushMeter.mark( containerList.size() );
+ Timer.Context time = flushTimer.time();
- subscriber.onNext( drainList );
+ execute( containerList );
- //take since we're in flight
- inFlight.addAndGet( drainList.size() );
-
-
- timer.stop();
- }
- //DO NOT add any doOnError* functions to this subscription. We want the producer
- //to receive these exceptions and sleep before a retry
- catch ( Throwable t ) {
- final long sleepTime = config.getFailureRetryTime();
-
- log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t );
-
- if ( drainList != null ) {
- inFlight.addAndGet( -1 * drainList.size() );
- }
-
- try {
- Thread.sleep( sleepTime );
- }
- catch ( InterruptedException ie ) {
- //swallow
- }
-
- indexErrorCounter.inc();
- }
- }
- while ( true );
- }
- } ).doOnNext( containerList -> {
- if ( containerList.size() == 0 ) {
- return;
- }
-
- flushMeter.mark(containerList.size());
- Timer.Context time = flushTimer.time();
-
-
- execute(containerList);
-
- time.stop();
- } )
+ time.stop();
+ } )
//ack after we process
- .doOnNext( indexOperationMessages -> {
- bufferQueue.ack( indexOperationMessages );
- //release so we know we've done processing
- inFlight.addAndGet( -1 * indexOperationMessages.size() );
- } ).subscribeOn( Schedulers.newThread() );
+ .doOnNext( indexOperationMessages -> {
- //start in the background
+ //release so we know we've done processing
+ inFlight.addAndGet( -1 * indexOperationMessages.size() );
+ } ).subscribe();
- final Subscription subscription = consumer.subscribe();
+ //start in the background
- subscriptions.add(subscription );
- }
}
/**
* Execute the request, check for errors, then re-init the batch for future use
*/
- private void execute( final List<IndexIdentifierImpl.IndexOperationMessage> operationMessages ) {
+ private void execute( final List<IndexOperationMessage> operationMessages ) {
if ( operationMessages == null || operationMessages.size() == 0 ) {
return;
@@ -250,28 +180,28 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.doOnNext( bulkRequestBuilder -> sendRequest( bulkRequestBuilder ) ).toBlocking().lastOrDefault( null );
//call back all futures
- Observable.from( operationMessages ).doOnNext( operationMessage -> operationMessage.getFuture().done() ).toBlocking().lastOrDefault( null );
+ Observable.from( operationMessages ).doOnNext( operationMessage -> operationMessage.getFuture().done() )
+ .toBlocking().lastOrDefault( null );
}
/**
* initialize request
- * @return
*/
private BulkRequestBuilder initRequest() {
BulkRequestBuilder bulkRequest = client.prepareBulk();
- bulkRequest.setConsistencyLevel(WriteConsistencyLevel.fromString(config.getWriteConsistencyLevel()));
- bulkRequest.setRefresh(config.isForcedRefresh());
+ bulkRequest.setConsistencyLevel( WriteConsistencyLevel.fromString( config.getWriteConsistencyLevel() ) );
+ bulkRequest.setRefresh( config.isForcedRefresh() );
return bulkRequest;
}
+
/**
* send bulk request
- * @param bulkRequest
*/
- private void sendRequest(BulkRequestBuilder bulkRequest) {
+ private void sendRequest( BulkRequestBuilder bulkRequest ) {
//nothing to do, we haven't added anything to the index
- if (bulkRequest.numberOfActions() == 0) {
+ if ( bulkRequest.numberOfActions() == 0 ) {
return;
}
@@ -280,9 +210,10 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
try {
responses = bulkRequest.execute().actionGet();
- } catch (Throwable t) {
- log.error("Unable to communicate with elasticsearch");
- failureMonitor.fail("Unable to execute batch", t);
+ }
+ catch ( Throwable t ) {
+ log.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@ -290,23 +221,43 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
boolean error = false;
- for (BulkItemResponse response : responses) {
+ for ( BulkItemResponse response : responses ) {
- if (response.isFailed()) {
+ if ( response.isFailed() ) {
// log error and continue processing
- log.error("Unable to index id={}, type={}, index={}, failureMessage={} ",
- response.getId(),
- response.getType(),
- response.getIndex(),
- response.getFailureMessage()
- );
+ log.error( "Unable to index id={}, type={}, index={}, failureMessage={} ", response.getId(),
+ response.getType(), response.getIndex(), response.getFailureMessage() );
error = true;
}
}
if ( error ) {
- throw new RuntimeException("Error during processing of bulk index operations one of the responses failed. Check previous log entries");
+ throw new RuntimeException(
+ "Error during processing of bulk index operations one of the responses failed. Check previous log "
+ + "entries" );
+ }
+ }
+
+
+ public static class BufferProducer implements Observable.OnSubscribe<IndexOperationMessage> {
+
+ private Subscriber<? super IndexOperationMessage> subscriber;
+
+
+ /**
+ * Send the data through the buffer
+ */
+ public void send( final IndexOperationMessage indexOp ) {
+
+ subscriber.onNext( indexOp );
+ }
+
+
+ @Override
+ public void call( final Subscriber<? super IndexOperationMessage> subscriber ) {
+ //just assigns for later use, doesn't do anything else
+ this.subscriber = subscriber;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
deleted file mode 100644
index 9ba92d0..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index.impl;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Timer;
-import com.google.common.base.Preconditions;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-
-
-/**
- * Producer for index operation messages
- */
-@Singleton
-public class EsIndexBufferProducerImpl implements IndexBufferProducer {
-
- private final Counter indexSizeCounter;
-
- private final Timer timer;
- private final BufferQueue bufferQueue;
-
- @Inject
- public EsIndexBufferProducerImpl( MetricsFactory metricsFactory, final BufferQueue bufferQueue ){
- this.bufferQueue = bufferQueue;
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferProducerImpl.class, "index.buffer.size");
- this.timer = metricsFactory.getTimer(EsIndexBufferProducerImpl.class,"index.buffer.producer.timer");
- }
-
- public BetterFuture put(IndexIdentifierImpl.IndexOperationMessage message){
- Preconditions.checkNotNull(message, "Message cannot be null");
- indexSizeCounter.inc(message.getDeIndexRequests().size());
- indexSizeCounter.inc(message.getIndexRequests().size());
- Timer.Context time = timer.time();
- bufferQueue.offer( message );
- time.stop();
- return message.getFuture();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
index fe31d35..fed9a50 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
@@ -98,30 +98,4 @@ public class FailureMonitorImpl implements FailureMonitor {
public void success() {
failCounter.set( 0 );
}
-
-
- /**
- * Identifier for where an index is in underlying server
- */
- public static interface IndexIdentifier {
-
- /**
- * get the alias name
- * @return
- */
- IndexAlias getAlias();
-
- /**
- * get index name from suffix
- * @param suffix
- * @return
- */
- String getIndex( String suffix );
-
- /**
- * return unique string
- * @return
- */
- String toString();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
new file mode 100644
index 0000000..4b84db7
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FlushBufferQueue.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+public class FlushBufferQueue {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
index b8e733d..3258444 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferConsumer.java
@@ -19,6 +19,10 @@
*/
package org.apache.usergrid.persistence.index.impl;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+
+
/**
* Buffer index requests
*/
@@ -26,12 +30,9 @@ public interface IndexBufferConsumer {
/**
- * Start the consumer
- */
- void start();
-
- /**
- * Stop the consumers
+ * Put this operation into our collapsing bufer
+ * @param message
+ * @return
*/
- void stop();
+ BetterFuture put(IndexOperationMessage message);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
deleted file mode 100644
index 36cb180..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexBufferProducer.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index.impl;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-
-
-/**
- * Produce to index buffer consumer
- */
-public interface IndexBufferProducer {
-
- BetterFuture put(IndexIdentifierImpl.IndexOperationMessage message);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
new file mode 100644
index 0000000..332c7b7
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifier.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+/**
+ * Identifier for where an index is in underlying server
+ */
+public interface IndexIdentifier {
+
+ /**
+ * get the alias name
+ * @return
+ */
+ IndexAlias getAlias();
+
+ /**
+ * get index name from suffix
+ * @param suffix
+ * @return
+ */
+ String getIndex( String suffix );
+
+ /**
+ * return unique string
+ * @return
+ */
+ String toString();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
index c47b2c8..782625b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
@@ -20,22 +20,15 @@
package org.apache.usergrid.persistence.index.impl;
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.Callable;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.inject.Inject;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.IndexFig;
/**
* Class is used to generate an index name and alias name
*/
-public class IndexIdentifierImpl implements FailureMonitorImpl.IndexIdentifier {
+public class IndexIdentifierImpl implements IndexIdentifier {
private final IndexFig config;
@Inject
@@ -71,113 +64,4 @@ public class IndexIdentifierImpl implements FailureMonitorImpl.IndexIdentifier {
public String toString() {
return "index id"+config.getIndexPrefix();
}
-
-
- /**
- * Container for index operations.
- */
- public static class IndexOperationMessage implements Serializable {
- private final Set<IndexRequest> indexRequests;
- private final Set<DeIndexRequest> deIndexRequests;
-
-
-
- private final BetterFuture<IndexOperationMessage> containerFuture;
-
-
- public IndexOperationMessage() {
- final IndexOperationMessage parent = this;
- this.indexRequests = new HashSet<>();
- this.deIndexRequests = new HashSet<>();
- this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
- @Override
- public IndexOperationMessage call() throws Exception {
- return parent;
- }
- } );
- }
-
-
- public void addIndexRequest( final IndexRequest indexRequest ) {
- indexRequests.add( indexRequest );
- }
-
-
- public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
- this.indexRequests.addAll( indexRequests );
- }
-
-
- public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
- this.deIndexRequests.add( deIndexRequest );
- }
-
-
- public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
- this.deIndexRequests.addAll( deIndexRequests );
- }
-
-
- public Set<IndexRequest> getIndexRequests() {
- return indexRequests;
- }
-
-
- public Set<DeIndexRequest> getDeIndexRequests() {
- return deIndexRequests;
- }
-
-
- @JsonIgnore
- public boolean isEmpty(){
- return indexRequests.isEmpty() && deIndexRequests.isEmpty();
- }
-
- /**
- * return the promise
- */
- @JsonIgnore
- public BetterFuture<IndexOperationMessage> getFuture() {
- return containerFuture;
- }
-
-
- @Override
- public boolean equals( final Object o ) {
- if ( this == o ) {
- return true;
- }
- if ( o == null || getClass() != o.getClass() ) {
- return false;
- }
-
- final IndexOperationMessage that = ( IndexOperationMessage ) o;
-
- if ( !deIndexRequests.equals( that.deIndexRequests ) ) {
- return false;
- }
- if ( !indexRequests.equals( that.indexRequests ) ) {
- return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode() {
- int result = indexRequests.hashCode();
- result = 31 * result + deIndexRequests.hashCode();
- return result;
- }
-
- public void done() {
- //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
- final BetterFuture<IndexOperationMessage> future = getFuture();
-
- if(future != null ){
- future.done();
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
new file mode 100644
index 0000000..1a60026
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+
+/**
+ * Container for index operations.
+ */
+public class IndexOperationMessage implements Serializable {
+ private final Set<IndexRequest> indexRequests;
+ private final Set<DeIndexRequest> deIndexRequests;
+
+
+
+ private final BetterFuture<IndexOperationMessage> containerFuture;
+
+
+ public IndexOperationMessage() {
+ final IndexOperationMessage parent = this;
+ this.indexRequests = new HashSet<>();
+ this.deIndexRequests = new HashSet<>();
+ this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
+ @Override
+ public IndexOperationMessage call() throws Exception {
+ return parent;
+ }
+ } );
+ }
+
+
+ public void addIndexRequest( final IndexRequest indexRequest ) {
+ indexRequests.add( indexRequest );
+ }
+
+
+ public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
+ this.indexRequests.addAll( indexRequests );
+ }
+
+
+ public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
+ this.deIndexRequests.add( deIndexRequest );
+ }
+
+
+ public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
+ this.deIndexRequests.addAll( deIndexRequests );
+ }
+
+
+ public Set<IndexRequest> getIndexRequests() {
+ return indexRequests;
+ }
+
+
+ public Set<DeIndexRequest> getDeIndexRequests() {
+ return deIndexRequests;
+ }
+
+
+ @JsonIgnore
+ public boolean isEmpty(){
+ return indexRequests.isEmpty() && deIndexRequests.isEmpty();
+ }
+
+ /**
+ * return the promise
+ */
+ @JsonIgnore
+ public BetterFuture<IndexOperationMessage> getFuture() {
+ return containerFuture;
+ }
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final IndexOperationMessage that = ( IndexOperationMessage ) o;
+
+ if ( !deIndexRequests.equals( that.deIndexRequests ) ) {
+ return false;
+ }
+ if ( !indexRequests.equals( that.indexRequests ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = indexRequests.hashCode();
+ result = 31 * result + deIndexRequests.hashCode();
+ return result;
+ }
+
+ public void done() {
+ //if this has been serialized, it could be null. don't NPE if it is, there's nothing to ack
+ final BetterFuture<IndexOperationMessage> future = getFuture();
+
+ if(future != null ){
+ future.done();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index f3663cd..5052ddf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -61,14 +61,14 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
private final IndexAlias alias;
private final IndexCache indexCache;
private final EsProvider esProvider;
- private final IndexBufferProducer producer;
+ private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer timer;
@Inject
- public IndexRefreshCommandImpl( FailureMonitorImpl.IndexIdentifier indexIdentifier, EsProvider esProvider,
- IndexBufferProducer producer, IndexFig indexFig, MetricsFactory metricsFactory,
+ public IndexRefreshCommandImpl( IndexIdentifier indexIdentifier, EsProvider esProvider,
+ IndexBufferConsumer producer, IndexFig indexFig, MetricsFactory metricsFactory,
final IndexCache indexCache ) {
@@ -105,7 +105,7 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
IndexRequest indexRequest = new IndexRequest( alias.getWriteAlias(), docId, entityData );
//save the item
- IndexIdentifierImpl.IndexOperationMessage message = new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage message = new IndexOperationMessage();
message.addIndexRequest( indexRequest );
producer.put( message );
@@ -153,8 +153,8 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
new DeIndexRequest( aliases, appScope, edge, entity.getId(), entity.getVersion() );
//delete the item
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage =
- new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage indexOperationMessage =
+ new IndexOperationMessage();
indexOperationMessage.addDeIndexRequest( deIndexRequest );
producer.put( indexOperationMessage );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
index c60de42..c93fd86 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
@@ -20,15 +20,15 @@
package org.apache.usergrid.persistence.index.migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.impl.FailureMonitorImpl;
import org.apache.usergrid.persistence.index.impl.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
/**
* Class is used to generate an index name and alias name the old way via app name
*/
-public class LegacyIndexIdentifier implements FailureMonitorImpl.IndexIdentifier {
+public class LegacyIndexIdentifier implements IndexIdentifier {
private final IndexFig config;
private final ApplicationScope applicationScope;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 20daf82..410f0e3 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,27 +19,17 @@
package org.apache.usergrid.persistence.index.guice;
-import com.google.inject.Inject;
-import com.google.inject.TypeLiteral;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.impl.IndexBufferProducer;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.IndexCache;
-import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import rx.Observable;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import java.util.UUID;
+import com.google.inject.Inject;
+import com.google.inject.TypeLiteral;
+
+import rx.Observable;
public class TestIndexModule extends TestModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
deleted file mode 100644
index 43e581a..0000000
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchType;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.guice.TestIndexModule;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-
-import com.google.inject.Inject;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
-@NotThreadSafe
-public class BufferQueueSQSImplTest {
-
-
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
-
-
- @Rule
- public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
-
- @Inject
- public QueueManagerFactory queueManagerFactory;
-
- @Inject
- public IndexFig indexFig;
-
- @Inject
- public MapManagerFactory mapManagerFactory;
-
- @Inject
- public MetricsFactory metricsFactory;
-
-
- private BufferQueueSQSImpl bufferQueueSQS;
-
- @Before
- public void setup(){
- bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
- }
-
-
-
-
- @Test
- public void testMessageIndexing(){
-
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
- final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
- assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
-
- final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
-
-
- final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
-
-
- //de-index request
- final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
-
-
- SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
-
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
- SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
-
-
-
-
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage = new IndexIdentifierImpl.IndexOperationMessage();
- indexOperationMessage.addIndexRequest( indexRequest1);
- indexOperationMessage.addIndexRequest( indexRequest2);
-
- indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
- indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
-
- bufferQueueSQS.offer( indexOperationMessage );
-
- //wait for it to send to SQS
- indexOperationMessage.getFuture().get();
-
- //now get it back
-
- final List<IndexIdentifierImpl.IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
-
- assertTrue(ops.size() > 0);
-
- final IndexIdentifierImpl.IndexOperationMessage returnedOperation = ops.get( 0 );
-
- //get the operations out
-
- final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
-
- assertTrue(indexRequestSet.contains(indexRequest1));
- assertTrue(indexRequestSet.contains(indexRequest2));
-
-
- final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
-
- assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
- assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
-
-
-
- //now ack the message
-
- bufferQueueSQS.ack( ops );
-
- }
-
- private List<IndexIdentifierImpl.IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-
- List<IndexIdentifierImpl.IndexOperationMessage> ops;
-
- do{
- ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
- }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
-
- return ops;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
deleted file mode 100644
index ba0dc1f..0000000
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.queue;
-
-
-import org.junit.Assume;
-import org.junit.internal.runners.model.MultipleFailureException;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-
-import com.amazonaws.AmazonClientException;
-
-
-/**
- * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
- * https://github.com/junit-team/junit/issues/116
- *
- * Until then, simply marks as passed, which is a bit dangerous
- */
-public class NoAWSCredsRule implements TestRule {
-
- public Statement apply( final Statement base, final Description description ) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
-
- try {
- base.evaluate();
- }
- catch ( Throwable t ) {
-
- if ( !isMissingCredsException( t ) ) {
- throw t;
- }
-
- //do this so our test gets marked as ignored. Not pretty, but it works
- Assume.assumeTrue( false );
-
-
- }
- }
- };
- }
-
-
- private boolean isMissingCredsException( final Throwable t ) {
-
- if ( t instanceof AmazonClientException ) {
-
- final AmazonClientException ace = ( AmazonClientException ) t;
-
- if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
- "could not get aws secret key from system properties" ) ) {
- //swallow
- return true;
- }
- }
-
- /**
- * Handle the multiple failure junit trace
- */
- if( t instanceof MultipleFailureException ){
- for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
- final boolean isMissingCreds = isMissingCredsException( failure );
-
- if(isMissingCreds){
- return true;
- }
- }
- }
- final Throwable cause = t.getCause();
-
- if ( cause == null ) {
- return false;
- }
-
-
- return isMissingCredsException( cause );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index eecb9e1..452d328 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -27,16 +27,15 @@ import java.util.List;
import java.util.Map;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 6074e65..2f1ccdf 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -284,42 +284,42 @@
<artifactId>slf4j-api</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-core</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-core</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
<dependency>
<groupId>${project.parent.groupId}</groupId>
- <artifactId>queue</artifactId>
+ <artifactId>common</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-config</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-config</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index e6970b5..fbf8290 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -16,33 +16,28 @@
*/
package org.apache.usergrid.services.notifications;
-import org.apache.commons.io.IOUtils;
-import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
-import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
-import org.apache.usergrid.persistence.entities.Notifier;
+import java.io.InputStream;
+import java.lang.reflect.Field;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
-import org.apache.usergrid.services.notifications.ConnectionException;
-import org.apache.usergrid.services.notifications.NotificationsService;
-import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.net.SocketException;
+import org.apache.commons.io.IOUtils;
+
import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.entities.Notifier;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.services.AbstractServiceIT;
import org.apache.usergrid.services.ServiceAction;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertArrayEquals;
-
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class NotifiersServiceIT extends AbstractServiceIT {
[35/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into USERGRID-578
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-578
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c77519ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c77519ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c77519ea
Branch: refs/heads/USERGRID-593
Commit: c77519ea78016b30ac9cb4957e4fc74a0093b983
Parents: 587e439 a58ccfa
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 14:33:10 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 14:33:10 2015 -0600
----------------------------------------------------------------------
.../index/AsyncIndexProvider.java | 5 +-
.../index/InMemoryAsyncReIndexService.java | 43 +++++--
.../corepersistence/index/IndexService.java | 12 +-
.../corepersistence/index/IndexServiceImpl.java | 55 ++++++---
.../corepersistence/TestIndexModule.java | 14 ++-
.../corepersistence/index/IndexServiceTest.java | 112 +++++++++++++++++++
.../corepersistence/index/PublishRxTest.java | 95 ++++++++++++++++
.../corepersistence/index/PublishRxtest.java | 61 ----------
.../usergrid/persistence/CollectionIT.java | 1 -
.../core/metrics/ObservableTimer.java | 72 ++++++++++++
.../core/rx/RxTaskSchedulerImpl.java | 5 +-
.../persistence/model/entity/Entity.java | 2 +-
.../persistence/index/impl/IndexRequest.java | 1 -
.../usergrid/cassandra/SpringResource.java | 13 ++-
14 files changed, 379 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c77519ea/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
index 0000000,973a42d..4e65110
mode 000000,100644..100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
@@@ -1,0 -1,95 +1,95 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.usergrid.corepersistence.index;
+
+
+ import java.util.concurrent.CountDownLatch;
+ import java.util.concurrent.TimeUnit;
+
+ import org.junit.Ignore;
+ import org.junit.Test;
+
+ import rx.Observable;
+ import rx.Subscription;
+ import rx.observables.ConnectableObservable;
+ import rx.schedulers.Schedulers;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
+
+
+ /**
+ * Test to test some assumptions about RX behaviors
+ */
-public class PublishRxTest {
++public class PublishRxtest {
+
+ @Test
+ public void testPublish() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final Subscription connectedObservable =
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ .subscribe();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();
+
+ assertTrue( "Subscription complete", completedSubscription );
+ }
+
+
+ @Test
+ @Ignore("This seems like it should work, yet blocks forever")
+ public void testConnectableObserver() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final ConnectableObservable<Integer> connectedObservable = Observable.range( 0, count ).publish();
+
+
+ //connect to our latch, which should run on it's own subscription
+ //start our latch running
+ connectedObservable.doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ final Observable<Integer> countObservable = connectedObservable.subscribeOn( Schedulers.io() ).count();
+
+ //start the sequence
+ connectedObservable.connect();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final int returnedCount = countObservable.toBlocking().last();
+
+ assertEquals( "Counts the same", count, returnedCount );
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c77519ea/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
[28/52] [abbrv] incubator-usergrid git commit: merge
Posted by to...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0830000e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0830000e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0830000e
Branch: refs/heads/USERGRID-593
Commit: 0830000e16282cdc6be4b9a8eb3e8f7b27ae4b7a
Parents: a0c2651
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 10:08:54 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 10:08:54 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/index/CandidateResults.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0830000e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 97ac813..7409a55 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -31,8 +31,6 @@ import com.google.common.base.Optional;
-import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
-
/**
* Internal results class, should not be returned as results to a user.
* Only returns candidate entity results
[20/52] [abbrv] incubator-usergrid git commit: scroll changes
Posted by to...@apache.org.
scroll changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f2714964
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f2714964
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f2714964
Branch: refs/heads/USERGRID-593
Commit: f27149649a47c153e831b46af87ad58b40faeb37
Parents: bbbe963
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Apr 20 17:33:12 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Apr 20 17:33:12 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsApplicationEntityIndexImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2714964/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 6b1f7d5..05318a2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -290,7 +290,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
candidates.add( candidateResult );
}
- final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),from+limit);
+ final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings());
// >= seems odd. However if we get an overflow, we need to account for it.
if ( length >= limit ) {
[15/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' into USERGRID-541
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-541
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/078666ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/078666ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/078666ec
Branch: refs/heads/USERGRID-593
Commit: 078666ec10904bc5eea6708882d4e20df4d66f83
Parents: 9b9a1b0 8d965b3
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 10:38:01 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 10:38:01 2015 -0600
----------------------------------------------------------------------
.gitattributes | 17 +
.gitignore | 4 +
.usergridversion | 1 +
CHANGELOG | 31 +
NOTICE | 17 +-
docs/.gitignore | 2 +
docs/Makefile | 177 +
docs/README | 31 +
docs/_theme/sphinx_rtd_theme/__init__.py | 17 +
docs/_theme/sphinx_rtd_theme/breadcrumbs.html | 19 +
docs/_theme/sphinx_rtd_theme/footer.html | 33 +
docs/_theme/sphinx_rtd_theme/layout.html | 162 +
docs/_theme/sphinx_rtd_theme/layout_old.html | 205 +
docs/_theme/sphinx_rtd_theme/search.html | 50 +
docs/_theme/sphinx_rtd_theme/searchbox.html | 7 +
.../sphinx_rtd_theme/static/css/badge_only.css | 2 +
.../sphinx_rtd_theme/static/css/theme.css | 5 +
.../static/fonts/FontAwesome.otf | Bin 0 -> 62856 bytes
.../static/fonts/fontawesome-webfont.eot | Bin 0 -> 38205 bytes
.../static/fonts/fontawesome-webfont.svg | 414 +
.../static/fonts/fontawesome-webfont.ttf | Bin 0 -> 80652 bytes
.../static/fonts/fontawesome-webfont.woff | Bin 0 -> 44432 bytes
docs/_theme/sphinx_rtd_theme/static/js/theme.js | 47 +
docs/_theme/sphinx_rtd_theme/theme.conf | 8 +
docs/_theme/sphinx_rtd_theme/versions.html | 37 +
docs/concepts/activity.md | 1121 ++
docs/concepts/applications.md | 78 +
docs/concepts/assets.md | 20 +
docs/concepts/collections.md | 21 +
docs/concepts/events-and-counters.md | 268 +
docs/concepts/groups.md | 38 +
docs/concepts/query-language.md | 427 +
docs/concepts/relationships.md | 100 +
docs/concepts/roles-and-permissions.md | 18 +
docs/concepts/users-devices.md | 108 +
docs/conf.py | 273 +
docs/contribute-code.md | 24 +
docs/deploy-local.md | 179 +
docs/getting-up-and-running-locally.md | 72 +
docs/index.md | 12 +
docs/lib/sphinxcontrib_markdown.py | 58 +
docs/make.bat | 242 +
docs/organizations-admins.md | 55 +
docs/presentations-and-videos.md | 7 +
docs/presentations-and-videos/presentations.md | 21 +
docs/presentations-and-videos/videos.md | 11 +
docs/upgrading.md | 3 +
docs/usage.md | 16 +
portal/Gruntfile.js | 20 +-
portal/README.md | 35 +-
portal/build.sh | 1 +
portal/config.js | 3 +-
portal/css/main.css | 23 +-
portal/img/sdk-sprites-large.png | Bin 14642 -> 15115 bytes
portal/img/sdk-sprites.png | Bin 5027 -> 4401 bytes
portal/index-template.html | 1 +
portal/js/global/help-service.js | 2 +-
portal/js/global/ug-service.js | 5 +-
portal/package.json | 2 +-
portal/pom.xml | 98 +
release/README.txt | 43 +
release/changelog.rb | 150 +
release/pom.xml | 65 -
release/release-candidate.sh | 296 +
release/release.sh | 248 +
release/sign-release.sh | 30 -
release/src/main/assembly/source.xml | 57 -
sdks/android/.gitignore | 20 +
sdks/android/NOTICE | 12 +
sdks/android/README.md | 33 +
sdks/android/README.txt | 2 -
sdks/android/assembly.xml | 55 +
sdks/android/build_release_zip.sh | 10 +
sdks/android/pom.xml | 76 +-
.../apache/usergrid/android/client/Client.java | 1036 --
.../client/callbacks/ApiResponseCallback.java | 26 -
.../client/callbacks/ClientAsyncTask.java | 54 -
.../client/callbacks/ClientCallback.java | 25 -
.../callbacks/DeviceRegistrationCallback.java | 25 -
.../callbacks/GroupsRetrievedCallback.java | 29 -
.../client/callbacks/QueryResultsCallback.java | 25 -
.../android/client/utils/DeviceUuidFactory.java | 170 -
.../android/client/utils/ObjectUtils.java | 36 -
.../usergrid/android/sdk/CounterIncrement.java | 72 +
.../sdk/DefaultURLConnectionFactory.java | 36 +
.../apache/usergrid/android/sdk/UGClient.java | 3179 +++++
.../android/sdk/URLConnectionFactory.java | 30 +
.../sdk/callbacks/ApiResponseCallback.java | 31 +
.../android/sdk/callbacks/ClientAsyncTask.java | 66 +
.../android/sdk/callbacks/ClientCallback.java | 31 +
.../sdk/callbacks/GroupsRetrievedCallback.java | 35 +
.../sdk/callbacks/QueryResultsCallback.java | 33 +
.../usergrid/android/sdk/entities/Activity.java | 1019 ++
.../android/sdk/entities/Collection.java | 338 +
.../usergrid/android/sdk/entities/Device.java | 122 +
.../usergrid/android/sdk/entities/Entity.java | 552 +
.../usergrid/android/sdk/entities/Group.java | 151 +
.../usergrid/android/sdk/entities/Message.java | 159 +
.../usergrid/android/sdk/entities/User.java | 315 +
.../android/sdk/exception/ClientException.java | 43 +
.../android/sdk/response/AggregateCounter.java | 58 +
.../sdk/response/AggregateCounterSet.java | 117 +
.../android/sdk/response/ApiResponse.java | 774 ++
.../sdk/response/ClientCredentialsInfo.java | 64 +
.../android/sdk/response/QueueInfo.java | 47 +
.../android/sdk/utils/DeviceUuidFactory.java | 173 +
.../usergrid/android/sdk/utils/JsonUtils.java | 185 +
.../usergrid/android/sdk/utils/MapUtils.java | 42 +
.../usergrid/android/sdk/utils/ObjectUtils.java | 39 +
.../usergrid/android/sdk/utils/UrlUtils.java | 127 +
sdks/dotnet/NOTICE | 56 +
.../NSubstitute.1.6.0.0.nupkg | Bin 619760 -> 0 bytes
.../NSubstitute.1.6.0.0.nuspec | 16 -
.../packages/NSubstitute.1.6.0.0/README.txt | 0
.../NSubstitute.1.6.0.0/acknowledgements.txt | 14 -
.../lib/NET35/NSubstitute.XML | 432 -
.../lib/NET35/NSubstitute.dll | Bin 1039360 -> 0 bytes
.../lib/NET40/NSubstitute.XML | 432 -
.../lib/NET40/NSubstitute.dll | Bin 561152 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nupkg | Bin 94702 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nuspec | 16 -
sdks/dotnet/packages/NUnit.2.6.2/README.txt | 3 -
.../NUnit.2.6.2/lib/nunit.framework.dll | Bin 147456 -> 0 bytes
.../NUnit.2.6.2/lib/nunit.framework.xml | 10915 -----------------
.../packages/Newtonsoft.Json.4.5.11/LICENSE.txt | 21 +
.../Newtonsoft.Json.4.5.11.nupkg | Bin 1409718 -> 0 bytes
.../Newtonsoft.Json.4.5.11.nuspec | 16 -
.../packages/Newtonsoft.Json.4.5.11/README.txt | 3 -
.../lib/net20/Newtonsoft.Json.dll | Bin 406016 -> 0 bytes
.../lib/net20/Newtonsoft.Json.xml | 8542 -------------
.../lib/net35/Newtonsoft.Json.dll | Bin 366592 -> 0 bytes
.../lib/net35/Newtonsoft.Json.xml | 7678 ------------
.../lib/net40/Newtonsoft.Json.dll | Bin 391680 -> 0 bytes
.../lib/net40/Newtonsoft.Json.xml | 7921 ------------
.../Newtonsoft.Json.dll | Bin 315904 -> 0 bytes
.../Newtonsoft.Json.xml | 7107 -----------
.../lib/sl3-wp/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl3-wp/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4-windowsphone71/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl4-windowsphone71/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4/Newtonsoft.Json.dll | Bin 337408 -> 0 bytes
.../lib/sl4/Newtonsoft.Json.xml | 7250 -----------
.../lib/winrt45/Newtonsoft.Json.dll | Bin 367616 -> 0 bytes
.../lib/winrt45/Newtonsoft.Json.xml | 7446 -----------
.../dotnet/packages/RestSharp.104.1/LICENSE.txt | 202 +
.../RestSharp.104.1/RestSharp.104.1.nupkg | Bin 605731 -> 0 bytes
.../RestSharp.104.1/RestSharp.104.1.nuspec | 16 -
.../lib/net35-client/RestSharp.dll | Bin 146944 -> 0 bytes
.../lib/net35-client/RestSharp.xml | 2627 ----
.../RestSharp.104.1/lib/net35/RestSharp.dll | Bin 146944 -> 0 bytes
.../RestSharp.104.1/lib/net35/RestSharp.xml | 2627 ----
.../lib/net4-client/RestSharp.dll | Bin 150016 -> 0 bytes
.../lib/net4-client/RestSharp.xml | 2696 ----
.../RestSharp.104.1/lib/net4/RestSharp.dll | Bin 150016 -> 0 bytes
.../RestSharp.104.1/lib/net4/RestSharp.xml | 2696 ----
.../lib/sl4-wp71/RestSharp.WindowsPhone.dll | Bin 165376 -> 0 bytes
.../lib/sl4-wp71/RestSharp.WindowsPhone.xml | 3510 ------
.../lib/sl4/RestSharp.Silverlight.dll | Bin 124928 -> 0 bytes
.../lib/sl4/RestSharp.Silverlight.xml | 2445 ----
sdks/html5-javascript/lib/modules/Client.js | 29 +-
sdks/nodejs/lib/usergrid.js | 31 +-
sdks/php/lib/vendor/Apache/Usergrid/Client.php | 33 +-
sdks/php5/apache-usergrid/.gitignore | 12 +
.../Examples/attrubites/attributes.php | 122 +
.../Examples/collections/books.php | 130 +
.../Examples/collections/data.php | 38 +
.../Examples/collections/users.php | 120 +
sdks/php5/apache-usergrid/Examples/examples.md | 17 +
sdks/php5/apache-usergrid/Examples/examples.php | 245 +
.../Examples/management/management.php | 91 +
.../apache-usergrid/Examples/messages/data.php | 17 +
.../Examples/messages/messages.php | 62 +
.../Examples/notifications/data.php | 17 +
.../Examples/notifications/notifications.php | 86 +
.../Examples/notifications/notifiers.php | 79 +
sdks/php5/apache-usergrid/LICENSE | 202 +
sdks/php5/apache-usergrid/README.md | 219 +
sdks/php5/apache-usergrid/TODO.md | 2 +
sdks/php5/apache-usergrid/composer.json | 40 +
sdks/php5/apache-usergrid/composer.lock | 1225 ++
sdks/php5/apache-usergrid/phpunit.xml | 29 +
.../src/Api/Exception/BadRequestException.php | 34 +
.../src/Api/Exception/InvalidIdException.php | 33 +
.../src/Api/Exception/NotFoundException.php | 33 +
.../src/Api/Exception/ServerErrorException.php | 33 +
.../src/Api/Exception/UnauthorizedException.php | 34 +
.../src/Api/Exception/UsergridException.php | 152 +
.../apache-usergrid/src/Api/Filters/Boolean.php | 44 +
.../apache-usergrid/src/Api/Filters/Date.php | 44 +
.../apache-usergrid/src/Api/Filters/Number.php | 48 +
.../apache-usergrid/src/Api/GuzzleClient.php | 64 +
.../src/Api/Models/Application.php | 35 +
.../src/Api/Models/BaseCollection.php | 116 +
.../src/Api/Models/Collection.php | 37 +
.../apache-usergrid/src/Api/Models/Device.php | 50 +
.../apache-usergrid/src/Api/Models/Entity.php | 36 +
.../apache-usergrid/src/Api/Models/Event.php | 35 +
.../apache-usergrid/src/Api/Models/Group.php | 54 +
.../src/Api/Models/GuzzleCommandTrait.php | 59 +
.../apache-usergrid/src/Api/Models/Notifier.php | 36 +
.../src/Api/Models/Organization.php | 38 +
.../apache-usergrid/src/Api/Models/Receipt.php | 36 +
.../apache-usergrid/src/Api/Models/Role.php | 54 +
.../apache-usergrid/src/Api/Models/User.php | 77 +
.../apache-usergrid/src/Api/QueryAggregator.php | 61 +
.../src/Api/ResourceIterator.php | 69 +
sdks/php5/apache-usergrid/src/Api/Usergrid.php | 452 +
.../Oauth2/GrantType/AuthorizationCode.php | 77 +
.../Oauth2/GrantType/ClientCredentials.php | 73 +
.../Oauth2/GrantType/GrantTypeInterface.php | 41 +
.../GrantType/HWIOAuthBundleRefreshToken.php | 62 +
.../Oauth2/GrantType/PasswordCredentials.php | 75 +
.../Plugin/Oauth2/GrantType/RefreshToken.php | 74 +
.../src/Guzzle/Plugin/Oauth2/Oauth2Plugin.php | 212 +
.../Laravel/ApacheUsergridServiceProvider.php | 159 +
.../src/Laravel/Facades/Usergrid.php | 42 +
.../src/Manifests/1.0.0/Application.php | 979 ++
.../src/Manifests/1.0.0/Errors.php | 53 +
.../src/Manifests/1.0.0/Management.php | 1076 ++
.../src/Manifests/1.0.0/Manifest.php | 25 +
.../src/Manifests/1.0.0/Notification.php | 154 +
.../src/Manifests/1.0.1/Activities.php | 24 +
.../src/Manifests/1.0.1/Application.php | 1108 ++
.../src/Manifests/1.0.1/Assets.php | 24 +
.../src/Manifests/1.0.1/Custom.php | 381 +
.../src/Manifests/1.0.1/Devices.php | 465 +
.../src/Manifests/1.0.1/Errors.php | 50 +
.../src/Manifests/1.0.1/Events.php | 24 +
.../src/Manifests/1.0.1/Groups.php | 486 +
.../src/Manifests/1.0.1/Management.php | 1076 ++
.../src/Manifests/1.0.1/Manifest.php | 24 +
.../src/Manifests/1.0.1/Messages.php | 19 +
.../src/Manifests/1.0.1/Notifications.php | 241 +
.../src/Manifests/1.0.1/Notifiers.php | 166 +
.../src/Manifests/1.0.1/Receipts.php | 24 +
.../src/Manifests/1.0.1/Roles.php | 381 +
.../src/Manifests/1.0.1/Users.php | 381 +
.../Manifests/swagger-json/applications.json | 1262 ++
.../src/Manifests/swagger-json/management.json | 1905 +++
.../src/Manifests/swagger-json/resources.json | 15 +
.../src/Native/ConfigRepository.php | 102 +
.../src/Native/Facades/Usergrid.php | 118 +
.../src/Native/UsergridBootstrapper.php | 150 +
sdks/php5/apache-usergrid/src/config/config.php | 53 +
.../tests/Api/ApplicationTest.php | 72 +
.../apache-usergrid/tests/Api/AttributeTest.php | 59 +
.../Api/Exception/BadRequestExceptionTest.php | 59 +
.../Api/Exception/InvalidIdExceptionTest.php | 58 +
.../Api/Exception/NotFoundExceptionTest.php | 57 +
.../Api/Exception/ServerErrorExceptionTest.php | 58 +
.../Api/Exception/UnauthorizedExceptionTest.php | 58 +
.../tests/Api/Filters/BooleanTest.php | 45 +
.../tests/Api/Filters/DateTest.php | 41 +
.../tests/Api/ManagementTest.php | 59 +
.../tests/Api/QueryAggregatorTest.php | 40 +
.../apache-usergrid/tests/Api/UsergridTest.php | 153 +
.../tests/Laravel/Facades/UsergridTest.php | 53 +
sdks/php5/apache-usergrid/tests/bootstrap.php | 61 +
stack/build-tools/pom.xml | 5 +-
stack/config/pom.xml | 1 +
.../main/resources/usergrid-default.properties | 37 +-
stack/core/pom.xml | 1 +
.../apache/usergrid/batch/job/package-info.java | 17 -
.../usergrid/batch/repository/package-info.java | 17 -
.../usergrid/batch/service/package-info.java | 17 -
.../org/apache/usergrid/count/package-info.java | 18 -
.../java/org/apache/usergrid/utils/BCrypt.java | 627 -
.../org/apache/usergrid/utils/package-info.java | 17 -
.../main/resources/usergrid-core-context.xml | 41 +-
.../apache/usergrid/batch/job/package-info.java | 17 -
.../org/apache/usergrid/utils/package-info.java | 17 -
.../collection/util/EntityHelper.java | 2 +-
.../persistence/index/CandidateResults.java | 9 +-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 17 +-
stack/launcher/pom.xml | 8 +
.../java/org/apache/usergrid/launcher/App.java | 7 +-
.../launcher/ClasspathStaticHttpHandler.java | 321 -
.../org/apache/usergrid/launcher/Server.java | 5 +-
.../apache/usergrid/launcher/package-info.java | 17 -
stack/mongo-emulator/pom.xml | 3 +-
stack/pom.xml | 10 +-
stack/rest/pom.xml | 1 -
.../rest/applications/ServiceResource.java | 13 +-
.../rest/management/users/UsersResource.java | 14 +-
.../applications/assets/AssetResourceIT.java | 106 +
stack/services/pom.xml | 6 +
.../management/AccountCreationProps.java | 11 +
.../usergrid/management/ManagementService.java | 2 +
.../cassandra/AccountCreationPropsImpl.java | 8 +
.../cassandra/ManagementServiceImpl.java | 30 +-
.../security/crypto/command/BcryptCommand.java | 4 +-
.../security/providers/AbstractProvider.java | 5 +-
.../providers/PingIdentityProvider.java | 19 +-
stack/tools/pom.xml | 1 +
.../org/apache/usergrid/tools/ExportAdmins.java | 236 +
.../org/apache/usergrid/tools/ImportAdmins.java | 451 +
.../apache/usergrid/tools/ResetSuperUser.java | 18 +-
stack/websocket/pom.xml | 1 +
299 files changed, 31666 insertions(+), 91709 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --cc stack/core/src/main/resources/usergrid-core-context.xml
index f33586f,37a2530..c5df523
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@@ -92,17 -96,12 +96,16 @@@
<property orgAppName="maxAttempts" value="10"/>
</bean> -->
-
<bean id="injector"
class="org.apache.usergrid.corepersistence.GuiceFactory">
- <constructor-arg ref="cassandraHostConfigurator" />
- <constructor-arg ref="properties" />
</bean>
+ <!-- <bean id="injector"
+ class="org.apache.usergrid.corepersistence.GuiceFactory">
+ <constructor-arg ref="cassandraHostConfigurator" />
+ <constructor-arg ref="properties" />
+ </bean>-->
+
<bean id="cassandraService"
class="org.apache.usergrid.persistence.cassandra.CassandraService" init-method="init" destroy-method="destroy">
<constructor-arg ref="properties" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/078666ec/stack/services/pom.xml
----------------------------------------------------------------------
[17/52] [abbrv] incubator-usergrid git commit: Removing references to
EntityManagerImpl to fix test runs.
Posted by to...@apache.org.
Removing references to EntityManagerImpl to fix test runs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8dafe7b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8dafe7b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8dafe7b5
Branch: refs/heads/USERGRID-593
Commit: 8dafe7b579e4db934c1d612a9b185022df9c4e1c
Parents: 2ae6a3d
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 15:29:37 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 15:29:37 2015 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-core-context.xml | 20 +-------------------
1 file changed, 1 insertion(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8dafe7b5/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c5df523..b406b78 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -164,26 +164,8 @@
<constructor-arg ref="properties"/>
</bean>
-
-
- <bean id="entityManager" class="org.apache.usergrid.persistence.cassandra.EntityManagerImpl" scope="prototype"/>
-
<bean id="mailUtils" class="org.apache.usergrid.utils.MailUtils" />
-
- <bean id="traceTagAspect" class="org.apache.usergrid.persistence.cassandra.util.TraceTagAspect"/>
-
- <aop:config>
- <aop:aspect id="traceParticipant" ref="traceTagAspect">
- <!-- pointcut on the prescense of the TraceParticipant annotation only -->
- <aop:pointcut id="emTraceParticipant"
- expression="execution(* org.apache.usergrid.persistence.cassandra.*EntityManagerImpl.*(..)) and
- @annotation(org.apache.usergrid.persistence.cassandra.util.TraceParticipant)"/>
- <aop:around
- pointcut-ref="emTraceParticipant"
- method="applyTrace"/>
- </aop:aspect>
- </aop:config>
-
+
<!-- ============================================================== -->
<!-- Scheduler Settings from removed Scheduler Module's app context -->
<!-- ============================================================== -->
[24/52] [abbrv] incubator-usergrid git commit: Start of rewire of the
rebuild process
Posted by to...@apache.org.
Start of rewire of the rebuild process
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3480a363
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3480a363
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3480a363
Branch: refs/heads/USERGRID-593
Commit: 3480a3637cc3cc6c79504642c3c44fbcf5878efd
Parents: ee1a66f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 19:52:51 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 19:52:51 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 +-
.../corepersistence/CpEntityManager.java | 85 +-----
.../corepersistence/CpEntityManagerFactory.java | 119 +++-----
.../corepersistence/CpRelationManager.java | 9 +-
.../index/AsyncIndexProvider.java | 12 +-
.../index/AsyncIndexService.java | 49 ----
.../index/AsyncReIndexService.java | 42 +++
.../index/InMemoryAsyncIndexService.java | 92 -------
.../index/InMemoryAsyncReIndexService.java | 89 ++++++
.../corepersistence/index/ReIndexAction.java | 33 +++
.../corepersistence/index/ReIndexService.java | 39 ++-
.../index/ReIndexServiceImpl.java | 43 ++-
.../index/SQSAsyncIndexService.java | 270 -------------------
.../index/SQSAsyncReIndexService.java | 269 ++++++++++++++++++
.../rx/impl/AllEntityIdsObservable.java | 5 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 4 +-
.../corepersistence/util/CpNamingUtils.java | 6 +-
.../usergrid/persistence/EntityManager.java | 5 -
.../persistence/EntityManagerFactory.java | 11 +-
.../index/SQSAsyncIndexServiceTest.java | 18 +-
.../PerformanceEntityRebuildIndexTest.java | 18 +-
.../cassandra/EntityManagerFactoryImplIT.java | 15 +-
.../graph/serialization/EdgesObservable.java | 5 +-
.../serialization/impl/EdgesObservableImpl.java | 6 +-
24 files changed, 584 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index bb936b5..51972d8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -22,7 +22,7 @@ import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.corepersistence.index.AsyncIndexProvider;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
import org.apache.usergrid.corepersistence.index.QueryFig;
@@ -37,9 +37,6 @@ import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
-import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
-import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -154,7 +151,7 @@ public class CoreModule extends AbstractModule {
bind( IndexService.class ).to( IndexServiceImpl.class );
//bind the queue provider
- bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
+ bind( AsyncReIndexService.class).toProvider( AsyncIndexProvider.class );
install( new GuicyFigModule( QueryFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6c3989d..72ca955 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,8 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -49,7 +48,6 @@ import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityFactory;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.Query;
@@ -68,7 +66,6 @@ import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -81,7 +78,6 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.map.MapManager;
@@ -179,7 +175,7 @@ public class CpEntityManager implements EntityManager {
private final CounterUtils counterUtils;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -219,7 +215,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncReIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -804,8 +800,8 @@ public class CpEntityManager implements EntityManager {
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
- Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields( Inflector.getInstance().singularize( collectionType ),
- Arrays.<Field>asList( uniqueLookupRepairField ) );
+ Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ), Arrays.<Field>asList( uniqueLookupRepairField ) );
if(fieldSetObservable == null){
logger.debug( "Couldn't return the observable based on unique entities." );
@@ -1736,7 +1732,7 @@ public class CpEntityManager implements EntityManager {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be);
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( roleName ), permission, timestamp );
+ getRolePermissionsKey( roleName ), permission, timestamp );
//Adding graphite metrics
Timer.Context timeRevokeRolePermission = entRevokeRolePermissionsTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -1747,8 +1743,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getRolePermissions( String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ),
- ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( roleName ) );
+ return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( roleName ) );
}
//TODO: does this need graphite monitoring
@@ -1830,8 +1826,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getGroupRolePermissions( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ),
- ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( groupId, roleName ) );
+ return cass.getAllColumnNames( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( groupId, roleName ) );
}
@@ -1840,7 +1836,7 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
removeFromDictionary( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ), DICTIONARY_ROLENAMES, roleName );
cass.deleteRow( cass.getApplicationKeyspace( applicationId ), ApplicationCF.ENTITY_DICTIONARIES,
- SimpleRoleRef.getIdForGroupIdAndRoleName( groupId, roleName ) );
+ SimpleRoleRef.getIdForGroupIdAndRoleName( groupId, roleName ) );
}
@@ -1931,7 +1927,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getGroupRoleRef( UUID groupId, String roleName ) throws Exception {
Results results = this.searchCollection( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
+ Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2830,64 +2826,7 @@ public class CpEntityManager implements EntityManager {
- /**
- * Completely reindex the named collection in the application associated with this EntityManager.
- */
- @Override
- public void reindexCollection(
- final EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
-
- CpWalker walker = new CpWalker( );
-
- walker.walkCollections( this, getApplication(), collectionName, reverse, new CpVisitor() {
-
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
-
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- // swallow this, it just means this was already updated, which accomplishes our task
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
- }
- } );
- }
-
-
- /**
- * Completely reindex the application associated with this EntityManager.
- */
- public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception {
-
- CpWalker walker = new CpWalker( );
-
- walker.walkCollections( this, getApplication(), null, false, new CpVisitor() {
-
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
- }
- } );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6d20b7..46db3f8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,24 +15,16 @@
*/
package org.apache.usergrid.corepersistence;
-import com.google.common.base.*;
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
@@ -41,11 +33,19 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.AbstractEntity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@ -59,7 +59,6 @@ import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -67,17 +66,27 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
-import rx.Observable;
-import java.util.*;
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+import rx.Observable;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
-import static org.apache.usergrid.persistence.Schema.*;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
/**
@@ -113,7 +122,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector) {
@@ -125,7 +134,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.indexService = injector.getInstance( AsyncIndexService.class );
+ this.indexService = injector.getInstance( AsyncReIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
@@ -307,14 +316,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
migrateAppInfo(applicationId,CpNamingUtils.DELETED_APPLICATION_INFO,CpNamingUtils.APPLICATION_INFO)
.toBlocking().lastOrDefault(null);
- this.rebuildApplicationIndexes(applicationId, new ProgressObserver() {
- @Override
- public void onProgress(EntityRef entity) {
- logger.info("Restored entity {}:{}", entity.getType(), entity.getUuid());
- }
- });
+ throw new UnsupportedOperationException( "Implement index rebuild" );
- return managementEm.get(new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO,applicationId));
+// return managementEm.get(new SimpleEntityRef(CpNamingUtils.APPLICATION_INFO,applicationId));
}
@Override
@@ -661,44 +665,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public void rebuildAllIndexes( ProgressObserver po ) throws Exception {
-
- logger.info("\n\nRebuilding all indexes\n");
-
- rebuildInternalIndexes( po );
-
- Map<String, UUID> appMap = getApplications();
-
- logger.info("About to rebuild indexes for {} applications", appMap.keySet().size());
-
- for ( UUID appUuid : appMap.values() ) {
- try {
- rebuildApplicationIndexes(appUuid, po);
- } catch ( Exception e) {
- logger.error("Error rebuilding index for app " + appUuid + " continuing...", e );
- }
- }
- }
-
-
- @Override
- public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
-
- // TODO: remove this after appinfo migration done
- rebuildApplicationIndexes( CpNamingUtils.SYSTEM_APP_ID, po);
- rebuildApplicationIndexes( CpNamingUtils.MANAGEMENT_APPLICATION_ID, po );
- }
-
-
- @Override
- public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
-
- EntityManager em = getEntityManager( appId );
- em.reindex( po );
-
- logger.info("\n\nRebuilt index for applicationId {} \n", appId);
- }
-
@Override
@@ -716,18 +682,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
- public void rebuildCollectionIndex(
- UUID appId, String collectionName, boolean reverse, ProgressObserver po ) throws Exception {
-
- EntityManager em = getEntityManager( appId );
-
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- Application app = em.getApplication();
-
- em.reindexCollection(po, collectionName, reverse);
-
- logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
- new Object[]{app.getName(), appId, collectionName});
+ public ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ) {
+
+ throw new UnsupportedOperationException( "Implement me" );
+//
+// EntityManager em = getEntityManager( appId );
+//
+// //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
+// Application app = em.getApplication();
+//
+// em.reindexCollection(po, collectionName, reverse);
+//
+// logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
+// new Object[]{app.getName(), appId, collectionName});
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index a3d8172..8f125ad 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,8 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
@@ -43,7 +42,6 @@ import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.RelationManager;
@@ -80,7 +78,6 @@ import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import rx.Observable;
-import rx.functions.Action1;
import rx.functions.Func1;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
@@ -121,13 +118,13 @@ public class CpRelationManager implements RelationManager {
private final ApplicationScope applicationScope;
- private final AsyncIndexService indexService;
+ private final AsyncReIndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncReIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
Assert.notNull( em, "Entity manager cannot be null" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index d00ef8e..2c48c13 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -35,7 +35,7 @@ import com.google.inject.Singleton;
* A provider to allow users to configure their queue impl via properties
*/
@Singleton
-public class AsyncIndexProvider implements Provider<AsyncIndexService> {
+public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
private final QueryFig queryFig;
@@ -46,7 +46,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final AllEntityIdsObservable allEntitiesObservable;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private AsyncIndexService asyncIndexService;
+ private AsyncReIndexService asyncIndexService;
@Inject
@@ -67,7 +67,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
@Override
@Singleton
- public AsyncIndexService get() {
+ public AsyncReIndexService get() {
if ( asyncIndexService == null ) {
asyncIndexService = getIndexService();
}
@@ -77,17 +77,17 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
}
- private AsyncIndexService getIndexService() {
+ private AsyncReIndexService getIndexService() {
final String value = queryFig.getQueueImplementation();
final Implementations impl = Implementations.valueOf( value );
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, rxTaskScheduler,
+ return new InMemoryAsyncReIndexService( indexService, rxTaskScheduler,
entityCollectionManagerFactory );
case SQS:
- return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ return new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
deleted file mode 100644
index 8b5ced1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * Low level queue service for indexing entities
- */
-public interface AsyncIndexService extends ReIndexService.IndexAction {
-
-
- /**
- * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
- * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
- * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
- * @param applicationScope
- * @param entity The entity to index
- */
- void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
new file mode 100644
index 0000000..c6eedd7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface AsyncReIndexService extends ReIndexAction {
+
+
+ /**
+ * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
+ * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
+ * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
+ * @param applicationScope
+ * @param entity The entity to index
+ */
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
deleted file mode 100644
index 0efb964..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-import rx.Observable;
-
-
-@Singleton
-public class InMemoryAsyncIndexService implements AsyncIndexService {
-
- private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncIndexService.class);
- private final IndexService indexService;
- private final RxTaskScheduler rxTaskScheduler;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
-
- @Inject
- public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.indexService = indexService;
- this.rxTaskScheduler = rxTaskScheduler;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
-
- //process the entity immediately
- //only process the same version, otherwise ignore
-
- Observable.just( toIndex ).doOnNext( entity -> {
- log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
- indexService.indexEntity( applicationScope, entity );
- } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
- }
-
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
-
- final Id entityId = entityIdScope.getId();
-
- final Entity
- entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
- entityId ).toBlocking().lastOrDefault( null );
-
-
- if(entity == null){
- log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
- }
-
- indexService.indexEntity(applicationScope, entity );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
new file mode 100644
index 0000000..5ebda87
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+
+
+@Singleton
+public class InMemoryAsyncReIndexService implements AsyncReIndexService {
+
+ private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncReIndexService.class);
+ private final IndexService indexService;
+ private final RxTaskScheduler rxTaskScheduler;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public InMemoryAsyncReIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.indexService = indexService;
+ this.rxTaskScheduler = rxTaskScheduler;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+
+ Observable.just( toIndex ).doOnNext( entity -> {
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
+ }
+
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+
+ final Id entityId = entityIdScope.getId();
+
+ final Entity
+ entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
+ entityId ).toBlocking().lastOrDefault( null );
+
+
+ if(entity == null){
+ log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
+ }
+
+ indexService.indexEntity( applicationScope, entity );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
new file mode 100644
index 0000000..086b2aa
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+
+
+/**
+ * Callback to perform an index operation based on an scope during bulk re-index operations
+ */
+@FunctionalInterface
+public interface ReIndexAction {
+
+ void index( final EntityIdScope entityIdScope );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index dca6cac..91409fe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,10 +20,12 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.FutureTask;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -31,6 +33,7 @@ import com.google.common.base.Optional;
import rx.Observable;
import rx.Observer;
+import rx.observables.ConnectableObservable;
/**
@@ -40,14 +43,14 @@ public interface ReIndexService {
/**
- * Reindex all applications using the cursor provided
- *
- * @param startTimestamp The timestamp to start seeking from
- *
- * @return a cursor that can be used to resume the operation on the next run
+ * Perform an index rebuild
+ * @param appId
+ * @param collection
+ * @return
*/
- IndexResponse reIndex( final rx.Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
- final Optional<Long> startTimestamp, final IndexAction indexAction );
+ IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> collectionName, final Optional<String> cursor,
+ final Optional<Long> startTimestamp );
+
/**
@@ -55,10 +58,10 @@ public interface ReIndexService {
*/
class IndexResponse {
final String cursor;
- final Observable<Long> indexedEdgecount;
+ final ConnectableObservable<EdgeScope> indexedEdgecount;
- public IndexResponse( final String cursor, final Observable<Long> indexedEdgecount ) {
+ public IndexResponse( final String cursor, final ConnectableObservable<EdgeScope> indexedEdgecount ) {
this.cursor = cursor;
this.indexedEdgecount = indexedEdgecount;
}
@@ -74,23 +77,13 @@ public interface ReIndexService {
/**
- * Return the observable long count of all edges indexed
+ * Return the observable of all edges to be indexed.
+ *
+ * Note that after subscribing "connect" will need to be called to ensure that processing begins
* @return
*/
- public Observable<Long> getCount() {
+ public ConnectableObservable<EdgeScope> getCount() {
return indexedEdgecount;
}
}
-
-
-
-
- /**
- * Callback to perform an index operation based on an scope during bulk re-index operations
- */
- @FunctionalInterface
- interface IndexAction {
-
- void index( final EntityIdScope entityIdScope );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 5c022e1..3553c87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -20,8 +20,10 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -37,11 +39,16 @@ import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
import rx.Observable;
import rx.observables.ConnectableObservable;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
+
+@Singleton
public class ReIndexServiceImpl implements ReIndexService {
private static final MapScope RESUME_MAP_SCOPTE =
@@ -51,48 +58,52 @@ public class ReIndexServiceImpl implements ReIndexService {
private static final int INDEX_TTL = 60 * 60 * 24 * 10;
+ private final AllApplicationsObservable allApplicationsObservable;
private final AllEntityIdsObservable allEntityIdsObservable;
private final QueryFig queryFig;
private final RxTaskScheduler rxTaskScheduler;
private final MapManager mapManager;
+ private final AsyncReIndexService indexService;
+ @Inject
public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
- final MapManagerFactory mapManagerFactory, final QueryFig queryFig,
- final RxTaskScheduler rxTaskScheduler ) {
+ final MapManagerFactory mapManagerFactory,
+ final AllApplicationsObservable allApplicationsObservable, final QueryFig queryFig,
+ final RxTaskScheduler rxTaskScheduler, final AsyncReIndexService indexService ) {
this.allEntityIdsObservable = allEntityIdsObservable;
+ this.allApplicationsObservable = allApplicationsObservable;
this.queryFig = queryFig;
this.rxTaskScheduler = rxTaskScheduler;
+ this.indexService = indexService;
this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPTE );
}
- @Override
- public IndexResponse reIndex( final Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
- final Optional<Long> startTimestamp, final IndexAction indexAction ) {
+ @Override
+ public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection,
+ final Optional<String> collectionName, final Optional<String> cursor,
+ final Optional<Long> startTimestamp ) {
//load our last emitted Scope if a cursor is present
if ( cursor.isPresent() ) {
throw new UnsupportedOperationException( "Build this" );
}
+
+ final Observable<ApplicationScope> applicationScopes = appId.isPresent()? Observable.just( getApplicationScope(appId.get()) ) : allApplicationsObservable.getData();
+
final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
//create an observable that loads each entity and indexes it, start it running with publish
final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, startTimestamp )
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, collectionName, startTimestamp )
//for each edge, create our scope and index on it
- .doOnNext( edge -> indexAction
- .index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) )
+ .doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
- .subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).publish();
-
-
- //count our longs
- final Observable<Long> indexedCount = runningReIndex.countLong();
//start our sampler and state persistence
@@ -107,7 +118,11 @@ public class ReIndexServiceImpl implements ReIndexService {
} ).subscribe();
- return new IndexResponse( newCursor, indexedCount );
+ //start pushing to both
+ runningReIndex.connect();
+
+
+ return new IndexResponse( newCursor, runningReIndex );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
deleted file mode 100644
index 6d06637..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class SQSAsyncIndexService implements AsyncIndexService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( SQSAsyncIndexService.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final QueryFig queryFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public SQSAsyncIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
- final MetricsFactory metricsFactory ) {
- final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.queryFig = queryFig;
-
- this.writeTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- public void offer( final IndexEntityEvent operation ) {
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
- final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if ( messages.size() == 0 ) {
- return Collections.emptyList();
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String payload = getBody( message );
-
- //now see if the key was there
-
-
- final IndexEntityEvent messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- public void ack( final List<IndexEntityEvent> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexEntityEvent ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- /** Read the object from Base64 string. */
- private IndexEntityEvent fromString( String s ) throws IOException {
- IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexEntityEvent o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
-
- private String getBody( final QueueMessage message ) {
- return message.getBody().toString();
- }
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- }
-
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexEntityEvent {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
- super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
- this.message = message;
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
- throw new UnsupportedOperationException( "Implement me" );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
new file mode 100644
index 0000000..60a804c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class SQSAsyncReIndexService implements AsyncReIndexService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( SQSAsyncReIndexService.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public SQSAsyncReIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ this.writeTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ public void offer( final IndexEntityEvent operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+ final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if ( messages.size() == 0 ) {
+ return Collections.emptyList();
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String payload = getBody( message );
+
+ //now see if the key was there
+
+
+ final IndexEntityEvent messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public void ack( final List<IndexEntityEvent> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexEntityEvent ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexEntityEvent fromString( String s ) throws IOException {
+ IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexEntityEvent o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+
+ private String getBody( final QueueMessage message ) {
+ return message.getBody().toString();
+ }
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ }
+
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexEntityEvent {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
+ super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
+ this.message = message;
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
+ throw new UnsupportedOperationException( "Implement index rebuild" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
index c805a59..b9e5373 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -43,9 +43,10 @@ public interface AllEntityIdsObservable {
/**
* Get all edges that represent edges to entities in the system
* @param appScopes
- * @param startTime The time to
+ * @param edgeType The edge type to use (if specified)
+ * @param startTime The time to start with
* @return
*/
- Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<Long> startTime);
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index f9df1f5..257fab1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -81,12 +81,12 @@ public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
@Override
- public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<Long> startTime) {
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<String> edgeType, final Optional<Long> startTime) {
return appScopes.flatMap( applicationScope -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(), startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
+ return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(),edgeType, startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 67cc0ca..c42ad10 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -205,9 +205,7 @@ public class CpNamingUtils {
public static ApplicationScope getApplicationScope( UUID applicationId ) {
// We can always generate a scope, it doesn't matter if the application exists yet or not.
- final ApplicationScopeImpl scope = new ApplicationScopeImpl( generateApplicationId( applicationId ) );
-
- return scope;
+ return new ApplicationScopeImpl( generateApplicationId( applicationId ) );
}
@@ -229,6 +227,8 @@ public class CpNamingUtils {
return generateApplicationId( MANAGEMENT_APPLICATION_ID );
}
+
+
/**
* Get the map scope for the applicationId to store entity uuid to type mapping
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index fc8b3d5..23b6d6b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -692,11 +692,6 @@ public interface EntityManager {
/** For testing purposes */
public void flushManagerCaches();
- void reindexCollection(
- EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception;
-
- public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
-
public Entity getUniqueEntityFromAlias( String aliasType, String aliasValue );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index e70cd0d..b3f4b62 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -21,6 +21,8 @@ import java.util.Map;
import java.util.UUID;
import com.google.common.base.Optional;
+
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.IndexRefreshCommand;
import rx.Observable;
@@ -163,12 +165,6 @@ public interface EntityManagerFactory {
public IndexRefreshCommand.IndexRefreshCommandInfo refreshIndex();
- public void rebuildAllIndexes( ProgressObserver po ) throws Exception;
-
- public void rebuildInternalIndexes( ProgressObserver po ) throws Exception;
-
- public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception;
-
/**
* Perform a realtime count of every entity in the system. This can be slow as it traverses the entire system graph
@@ -178,8 +174,7 @@ public interface EntityManagerFactory {
/** For testing purposes */
public void flushEntityManagerCaches();
- void rebuildCollectionIndex(
- UUID appId, String collection, boolean reverse, ProgressObserver po) throws Exception;
+ ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection );
/**
* Add a new index to the application for scale
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
index 0977caa..5e9a5a1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
@@ -20,19 +20,9 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
import org.apache.usergrid.corepersistence.TestIndexModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.index.impl.IndexRequest;
-import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
+
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -41,10 +31,8 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
import com.google.inject.Inject;
@@ -80,11 +68,11 @@ public class SQSAsyncIndexServiceTest {
public MetricsFactory metricsFactory;
- private SQSAsyncIndexService bufferQueueSQS;
+ private SQSAsyncReIndexService bufferQueueSQS;
@Before
public void setup(){
- bufferQueueSQS = new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ bufferQueueSQS = new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index b0fc245..a293a0c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -213,11 +213,12 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- // do it forwards
- setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
-
- // and backwards, just to make sure both cases are covered
- setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", true, po );
+ fail( "Implement index rebuild" );
+// // do it forwards
+// setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
+//
+// // and backwards, just to make sure both cases are covered
+// setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", true, po );
reporter.report();
registry.remove( meterName );
@@ -354,9 +355,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- setup.getEmf().rebuildInternalIndexes( po );
-
- setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+ fail( "Implement index rebuild" );
+// setup.getEmf().rebuildInternalIndexes( po );
+//
+// setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
reporter.report();
registry.remove( meterName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index 1a0a13b..2452606 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -52,6 +52,8 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
@NotThreadSafe
public class EntityManagerFactoryImplIT extends AbstractCoreIT {
@@ -175,12 +177,13 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
// restore the app
emf.restoreApplication(deletedAppId);
- emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
- @Override
- public void onProgress(EntityRef entity) {
- logger.debug("Reindexing {}:{}", entity.getType(), entity.getUuid());
- }
- });
+ fail( "Implement index rebuild" );
+// emf.rebuildAllIndexes(new EntityManagerFactory.ProgressObserver() {
+// @Override
+// public void onProgress(EntityRef entity) {
+// logger.debug("Reindexing {}:{}", entity.getType(), entity.getUuid());
+// }
+// });
this.app.refreshIndex();
// test to see that app now works and is happy
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 9e7b8e6..9f0bd60 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -45,10 +45,11 @@ public interface EdgesObservable {
* Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
* @param gm
* @param sourceNode
- * @param startTimestamp
+ * @param edgeType The edge type if specified. Otherwise all types will be used
+ * @param startTimestamp The start timestamp if specfiied, otherwise Long.MIN will be used
* @return
*/
- Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,final Optional<String> edgeType,
final Optional<Long> startTimestamp );
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3480a363/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 859ca2e..ca9fb03 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -71,10 +71,12 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
- public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode, final Optional<String> edgeTypeInput,
final Optional<Long> startTimestamp ) {
- final Observable<String> edgeTypes =
+
+
+ final Observable<String> edgeTypes = edgeTypeInput.isPresent()? Observable.just( edgeTypeInput.get() ):
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
[23/52] [abbrv] incubator-usergrid git commit: Fixes several wiring
bugs
Posted by to...@apache.org.
Fixes several wiring bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ee1a66fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ee1a66fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ee1a66fa
Branch: refs/heads/USERGRID-593
Commit: ee1a66fabd61f032a8748db2d5914171fd8ae59a
Parents: 1c207d7
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 19:09:46 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 19:09:46 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 7 ++-
.../corepersistence/index/IndexService.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 13 +++--
.../rx/impl/AllEntityIdsObservableImpl.java | 5 ++
.../corepersistence/index/PublishRxtest.java | 61 ++++++++++++++++++++
.../index/impl/EsEntityIndexImpl.java | 2 +-
6 files changed, 84 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index f49e066..bb936b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -34,6 +34,8 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
@@ -102,7 +104,7 @@ public class CoreModule extends AbstractModule {
@Override
public void configureMigrationProvider() {
bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
- AllApplicationsObservable.class );
+ AllApplicationsObservableImpl.class );
}
});
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
@@ -141,6 +143,7 @@ public class CoreModule extends AbstractModule {
plugins.addBinding().to( MigrationModuleVersionPlugin.class );
bind( AllApplicationsObservable.class ).to( AllApplicationsObservableImpl.class );
+ bind( AllEntityIdsObservable.class).to( AllEntityIdsObservableImpl.class );
/*****
@@ -148,7 +151,7 @@ public class CoreModule extends AbstractModule {
*****/
- bind(IndexService.class).to( IndexServiceImpl.class );
+ bind( IndexService.class ).to( IndexServiceImpl.class );
//bind the queue provider
bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index 2bf073c..41fe323 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -41,9 +41,11 @@ public interface IndexService {
* @param applicationScope The scope of the entity
* @param entity The entity
*
- * @return An observable with the count of every
+ * @return An observable with the count of every batch executed to index the entity. Note that this a cold observable
+ * and must be subscribed to in order to perform the operation. This also makes no assumptions on scheduling. It is up to the caller
+ * to assign the correct scheduler to the observable
*/
- Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity );
+ Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 873e2b6..29dd04a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.Collection;
+
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.graph.Edge;
@@ -40,6 +42,7 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
+import rx.observables.ConnectableObservable;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeToTarget;
@@ -69,7 +72,7 @@ public class IndexServiceImpl implements IndexService {
@Override
- public Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
+ public Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
//bootstrap the lower modules from their caches
@@ -93,7 +96,8 @@ public class IndexServiceImpl implements IndexService {
final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
- final Observable<IndexOperationMessage> observable =
+ //start the observable via publish
+ final ConnectableObservable<IndexOperationMessage> observable =
//try to send a whole batch if we can
Observable.merge( sourceEdgesToIndex, targetSizes ).buffer( indexFig.getIndexBatchSize() )
@@ -101,12 +105,11 @@ public class IndexServiceImpl implements IndexService {
.flatMap( buffer -> Observable.from( buffer ).collect( () -> ei.createBatch(),
( batch, indexEdge ) -> batch.index( indexEdge, entity ) )
//return the future from the batch execution
- .flatMap( batch -> Observable.from( batch.execute() ) ) );
+ .flatMap( batch -> Observable.from( batch.execute() ) ) ).publish();
- observable.toBlocking().last();
- return Observable.just( 0 );
+ return observable.countLong();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
index 1f74420..f9df1f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -21,7 +21,10 @@ package org.apache.usergrid.corepersistence.rx.impl;
+
import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -37,12 +40,14 @@ import rx.Observable;
/**
* An implementation that will provide all entityId scopes in the system
*/
+@Singleton
public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
private final GraphManagerFactory graphManagerFactory;
private final TargetIdObservable targetIdObservable;
private final EdgesObservable edgesObservable;
+ @Inject
public AllEntityIdsObservableImpl( final GraphManagerFactory graphManagerFactory,
final TargetIdObservable targetIdObservable,
final EdgesObservable edgesObservable ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
new file mode 100644
index 0000000..1eb00c2
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+
+import rx.Observable;
+import rx.Subscription;
+import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Validates the Rx scheduler works as expected with publish
+ */
+public class PublishRxTest {
+
+ @Test
+ public void testPublish() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final Subscription connectedObservable =
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ .subscribe();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();;
+
+ assertTrue("Subscription complete", completedSubscription);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee1a66fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fcfba15..dcda045 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -275,7 +275,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
// delete all types, this way if we miss one it will get cleaned up
esProvider.getClient().prepareDelete( alias.getWriteAlias(), VERIFY_TYPE, tempId).get();
- logger.info( "Successfully deleted documents in index {} read {} write {} and type {} with id {}",
+ logger.info( "Successfully deleted documents in read {} write {} and type {} with id {}",
alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE, tempId );
return true;
[36/52] [abbrv] incubator-usergrid git commit: Removed unused CFs
Posted by to...@apache.org.
Removed unused CFs
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2a582700
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2a582700
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2a582700
Branch: refs/heads/USERGRID-593
Commit: 2a582700b1f8bf68b3773a91a101c88de9db2b87
Parents: a58ccfa
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Apr 21 14:35:02 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Apr 21 14:35:02 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/mq/cassandra/QueuesCF.java | 5 -
.../persistence/cassandra/ApplicationCF.java | 37 ------
.../cassandra/CassandraPersistenceUtils.java | 122 +------------------
3 files changed, 1 insertion(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2a582700/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueuesCF.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueuesCF.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueuesCF.java
index a8b1166..b0f6906 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueuesCF.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueuesCF.java
@@ -51,8 +51,6 @@ public enum QueuesCF implements CFEnum {
CONSUMERS( "MQ_Consumers", "BytesType" ),
- CONSUMER_QUEUE_MESSAGES_PROPERTIES( "Consumer_Queue_Messages_Properties", "BytesType" ),
-
COUNTERS( "MQ_Counters", "BytesType", COUNTERTYPE.getClassName() ),
PROPERTY_INDEX( "MQ_Property_Index",
@@ -67,9 +65,6 @@ public enum QueuesCF implements CFEnum {
"I=>IntegerType(reversed=true),X=>LexicalUUIDType(reversed=true),L=>LongType(reversed=true)," +
"T=>TimeUUIDType(reversed=true),S=>UTF8Type(reversed=true),U=>UUIDType(reversed=true))" ),;
- public final static String STATIC_MESSAGES_KEYSPACE = "Usergrid_Messages";
- public final static String APPLICATION_MESSAGES_KEYSPACE_SUFFIX = "_messages";
-
private final String cf;
private final String comparator;
private final String validator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2a582700/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ApplicationCF.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ApplicationCF.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ApplicationCF.java
index 653c546..5dbe4b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ApplicationCF.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/ApplicationCF.java
@@ -27,19 +27,10 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
public enum ApplicationCF implements CFEnum {
- /** This is where the entity objects are stored */
- ENTITY_PROPERTIES( "Entity_Properties", "BytesType" ),
-
/** each row models name:value pairs. {@see org.apache.usergrid.persistence.Schema} for the list of dictionary types */
ENTITY_DICTIONARIES( "Entity_Dictionaries", "BytesType" ),
/**
- * Rows that are full of UUIDs. Used when we want to have a row full of references to other entities. Mainly, this
- * is for collections. Collections are represented by this CF.
- */
- ENTITY_ID_SETS( "Entity_Id_Sets", "UUIDType" ),
-
- /**
* Typed vs. untyped dictionary. Dynamic entity dictionaries end up here. {@link
* EntityManagerImpl#getDictionaryAsMap(org.apache.usergrid.persistence.EntityRef, String)}
*/
@@ -49,39 +40,11 @@ public enum ApplicationCF implements CFEnum {
"I=>IntegerType(reversed=true),X=>LexicalUUIDType(reversed=true),L=>LongType(reversed=true)," +
"T=>TimeUUIDType(reversed=true),S=>UTF8Type(reversed=true),U=>UUIDType(reversed=true))" ),
- /** No longer used? */
- ENTITY_METADATA( "Entity_Metadata", "BytesType" ),
-
- /** Contains all secondary indexes for entities */
- ENTITY_INDEX( "Entity_Index",
- "DynamicCompositeType(a=>AsciiType,b=>BytesType,i=>IntegerType,x=>LexicalUUIDType,l=>LongType," +
- "t=>TimeUUIDType,s=>UTF8Type,u=>UUIDType,A=>AsciiType(reversed=true),B=>BytesType(reversed=true)," +
- "I=>IntegerType(reversed=true),X=>LexicalUUIDType(reversed=true),L=>LongType(reversed=true)," +
- "T=>TimeUUIDType(reversed=true),S=>UTF8Type(reversed=true),U=>UUIDType(reversed=true))" ),
-
- /** Unique index for properties that must remain the same */
- ENTITY_UNIQUE( "Entity_Unique", "UUIDType" ),
-
- /** Contains all properties that have ever been indexed for an entity */
- ENTITY_INDEX_ENTRIES( "Entity_Index_Entries",
- "DynamicCompositeType(a=>AsciiType,b=>BytesType,i=>IntegerType,x=>LexicalUUIDType,l=>LongType," +
- "t=>TimeUUIDType,s=>UTF8Type,u=>UUIDType,A=>AsciiType(reversed=true),B=>BytesType(reversed=true)," +
- "I=>IntegerType(reversed=true),X=>LexicalUUIDType(reversed=true),L=>LongType(reversed=true)," +
- "T=>TimeUUIDType(reversed=true),S=>UTF8Type(reversed=true),U=>UUIDType(reversed=true))" ),
-
- /** All roles that exist within an application */
- APPLICATION_ROLES( "Application_Roles", "BytesType" ),
-
/** Application counters */
APPLICATION_AGGREGATE_COUNTERS( "Application_Aggregate_Counters", "LongType", COUNTERTYPE.getClassName() ),
/** Entity counters */
ENTITY_COUNTERS( "Entity_Counters", "BytesType", COUNTERTYPE.getClassName() ),;
- public final static String DEFAULT_DYNAMIC_COMPOSITE_ALIASES =
- "(a=>AsciiType,b=>BytesType,i=>IntegerType,x=>LexicalUUIDType,l=>LongType,t=>TimeUUIDType,s=>UTF8Type," +
- "u=>UUIDType,A=>AsciiType(reversed=true),B=>BytesType(reversed=true)," +
- "I=>IntegerType(reversed=true),X=>LexicalUUIDType(reversed=true),L=>LongType(reversed=true)," +
- "T=>TimeUUIDType(reversed=true),S=>UTF8Type(reversed=true),U=>UUIDType(reversed=true))";
private final String cf;
private final String comparator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2a582700/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
index 174e0a4..bbf844b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
@@ -83,16 +83,6 @@ public class CassandraPersistenceUtils {
/**
*
*/
- public static final ByteBuffer PROPERTY_TYPE_AS_BYTES = bytebuffer( PROPERTY_TYPE );
-
- /**
- *
- */
- public static final ByteBuffer PROPERTY_ID_AS_BYTES = bytebuffer( PROPERTY_UUID );
-
- /**
- *
- */
public static final char KEY_DELIM = ':';
/**
@@ -145,34 +135,6 @@ public class CassandraPersistenceUtils {
}
- public static void addPropertyToMutator( Mutator<ByteBuffer> m, Object key, String entityType, String propertyName,
- Object propertyValue, long timestamp ) {
-
- logBatchOperation( "Insert", ApplicationCF.ENTITY_PROPERTIES, key, propertyName, propertyValue, timestamp );
-
- HColumn<ByteBuffer, ByteBuffer> column = createColumn( bytebuffer( propertyName ),
- serializeEntityProperty( entityType, propertyName, propertyValue ), timestamp, be, be );
- m.addInsertion( bytebuffer( key ), ApplicationCF.ENTITY_PROPERTIES.toString(), column );
- }
-
-
- public static void addPropertyToMutator( Mutator<ByteBuffer> m, Object key, String entityType,
- Map<String, ?> columns, long timestamp ) throws Exception {
-
- for ( Entry<String, ?> entry : columns.entrySet() ) {
- addPropertyToMutator( m, key, entityType, entry.getKey(), entry.getValue(), timestamp );
- }
- }
-
-
- /** Delete the row */
- public static void addDeleteToMutator( Mutator<ByteBuffer> m, Object columnFamily, Object key, long timestamp )
- throws Exception {
-
- logBatchOperation( "Delete", columnFamily, key, null, null, timestamp );
-
- m.addDeletion( bytebuffer( key ), columnFamily.toString(), timestamp );
- }
public static void addDeleteToMutator( Mutator<ByteBuffer> m, Object columnFamily, Object key, Object columnName,
@@ -188,20 +150,6 @@ public class CassandraPersistenceUtils {
}
- public static void addDeleteToMutator( Mutator<ByteBuffer> m, Object columnFamily, Object key, long timestamp,
- Object... columnNames ) throws Exception {
-
- for ( Object columnName : columnNames ) {
- logBatchOperation( "Delete", columnFamily, key, columnName, null, timestamp );
-
- if ( columnName instanceof List<?> ) {
- columnName = DynamicComposite.toByteBuffer( ( List<?> ) columnName );
- }
-
- m.addDeletion( bytebuffer( key ), columnFamily.toString(), bytebuffer( columnName ), be, timestamp );
- }
- }
-
public static Map<String, ByteBuffer> getColumnMap( List<HColumn<String, ByteBuffer>> columns ) {
Map<String, ByteBuffer> column_map = new TreeMap<String, ByteBuffer>( String.CASE_INSENSITIVE_ORDER );
@@ -228,30 +176,6 @@ public class CassandraPersistenceUtils {
}
- public static List<ByteBuffer> getAsByteKeys( List<UUID> ids ) {
- List<ByteBuffer> keys = new ArrayList<ByteBuffer>();
- for ( UUID id : ids ) {
- keys.add( bytebuffer( key( id ) ) );
- }
- return keys;
- }
-
-
- /** @return timestamp value for current time */
- public static long createTimestamp() {
- return createClockResolution( ClockResolution.MICROSECONDS ).createClock();
- }
-
-
- /** @return normalized group path */
- public static String normalizeGroupPath( String path ) {
- path = replaceAll( path.toLowerCase().trim(), "//", "/" );
- path = removeStart( path, "/" );
- path = removeEnd( path, "/" );
- return path;
- }
-
-
/** @return a composite key */
public static Object key( Object... objects ) {
if ( objects.length == 1 ) {
@@ -304,30 +228,7 @@ public class CassandraPersistenceUtils {
return uuid;
}
-
- /** @return UUID for entity alias */
- public static UUID aliasID( UUID ownerId, String aliasType, String alias ) {
- return keyID( ownerId, aliasType, alias );
- }
-
-
- public static Mutator<ByteBuffer> buildSetIdListMutator( Mutator<ByteBuffer> batch, UUID targetId,
- String columnFamily, String keyPrefix, String keySuffix,
- List<UUID> keyIds, long timestamp ) throws Exception {
- for ( UUID keyId : keyIds ) {
- ByteBuffer key = null;
- if ( ( StringUtils.isNotEmpty( keyPrefix ) ) || ( StringUtils.isNotEmpty( keySuffix ) ) ) {
- key = bytebuffer( keyPrefix + keyId.toString() + keySuffix );
- }
- else {
- key = bytebuffer( keyId );
- }
- addInsertToMutator( batch, columnFamily, key, targetId, ByteBuffer.allocate( 0 ), timestamp );
- }
- return batch;
- }
-
- //No longer does retries
+ //No longer does retries
public static MutationResult batchExecute( Mutator<?> m, int retries ) {
return m.execute();
@@ -371,16 +272,6 @@ public class CassandraPersistenceUtils {
}
- public static ByteBuffer toStorableBinaryValue( Object obj ) {
- obj = toStorableValue( obj );
- if ( obj instanceof JsonNode ) {
- return JsonUtils.toByteBuffer( obj );
- }
- else {
- return bytebuffer( obj );
- }
- }
-
public static ByteBuffer toStorableBinaryValue( Object obj, boolean forceJson ) {
obj = toStorableValue( obj );
@@ -460,15 +351,4 @@ public class CassandraPersistenceUtils {
}
- public static void validateKeyspace( CFEnum[] cf_enums, KeyspaceDefinition ksDef ) {
- Map<String, ColumnFamilyDefinition> cfs = new HashMap<String, ColumnFamilyDefinition>();
- for ( ColumnFamilyDefinition cf : ksDef.getCfDefs() ) {
- cfs.put( cf.getName(), cf );
- }
- for ( CFEnum c : cf_enums ) {
- if ( !cfs.keySet().contains( c.getColumnFamily() ) ) {
-
- }
- }
- }
}
[46/52] [abbrv] incubator-usergrid git commit: adding background tasks
Posted by to...@apache.org.
adding background tasks
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/381d8c57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/381d8c57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/381d8c57
Branch: refs/heads/USERGRID-593
Commit: 381d8c57f2156ed3783a5d570cfdd92c7bfd5be1
Parents: 5594d74
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Apr 22 18:09:49 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Apr 22 18:09:49 2015 -0600
----------------------------------------------------------------------
.../Usergrid.Notifications/BackgroundTask.cs | 57 ++++++++++++++++++++
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 12 +++--
.../Usergrid.Notifications/MainPage.xaml.cs | 14 +++--
.../Usergrid.Notifications/MyBackgroundTask.cs | 18 +++++++
.../Usergrid.Notifications/Package.appxmanifest | 8 +++
.../Usergrid.Notifications.csproj | 1 +
.../Usergrid.Notifications.sln | 40 ++++++++++++++
.../persistence/query/IteratingQueryIT.java | 54 +++++++++----------
9 files changed, 170 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/BackgroundTask.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/BackgroundTask.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/BackgroundTask.cs
new file mode 100644
index 0000000..1096ca8
--- /dev/null
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/BackgroundTask.cs
@@ -0,0 +1,57 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Windows.ApplicationModel.Background;
+
+namespace Usergrid.Notifications
+{
+ public sealed class ExampleBackgroundTask : IBackgroundTask
+ {
+ private static BackgroundTaskRegistration task;
+ public void Run(IBackgroundTaskInstance taskInstance)
+ {
+ var test = "";
+
+ }
+ public static void Register()
+ {
+
+ foreach (var iter in BackgroundTaskRegistration.AllTasks)
+ {
+ IBackgroundTaskRegistration mytask = iter.Value;
+ if (mytask.Name == "ExampleBackgroundTask")
+ {
+
+ mytask.Unregister(true);
+ break;
+ }
+ }
+
+
+ var builder = new BackgroundTaskBuilder();
+ PushNotificationTrigger trigger = new PushNotificationTrigger();
+ builder.SetTrigger(trigger);
+ builder.Name = "ExampleBackgroundTask";
+ builder.TaskEntryPoint = "Usergrid.Notifications.ExampleBackgroundTask";
+
+ ExampleBackgroundTask.task = builder.Register();
+ task.Progress += task_Progress;
+
+ task.Completed += task_Completed;
+
+ }
+
+ static void task_Progress(BackgroundTaskRegistration sender, BackgroundTaskProgressEventArgs args)
+ {
+
+ var test = "done";
+ }
+
+ static void task_Completed(BackgroundTaskRegistration sender, BackgroundTaskCompletedEventArgs args)
+ {
+ var test = "done";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
index 5efbb9b..5279e32 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
@@ -108,7 +108,7 @@ namespace Usergrid.Notifications.Client
/// <param name="message"></param>
/// <returns></returns>
Task<bool> SendToast(string message);
- Task<bool> SendRaw(string message);
+ Task<bool> SendRaw(JObject message);
/// <summary>
/// Send a badge update
/// </summary>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
index 343e28b..54d7b00 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
@@ -86,7 +86,7 @@ namespace Usergrid.Notifications.Client
}
- public async Task<bool> SendRaw(String message)
+ public async Task<bool> SendRaw(JObject message)
{
if (DeviceId == null)
{
@@ -95,7 +95,7 @@ namespace Usergrid.Notifications.Client
var jsonObject = new JObject();
var payloads = new JObject();
var payload = new JObject();
- payload.Add("raw", new JValue(message));
+ payload.Add("raw", message);
payloads.Add(Notifier, payload);
jsonObject.Add("payloads", payloads);
jsonObject.Add("debug", true);
@@ -107,6 +107,8 @@ namespace Usergrid.Notifications.Client
{
channel = await PushNotificationChannelManager.CreatePushNotificationChannelForApplicationAsync().AsTask<PushNotificationChannel>();
channel.PushNotificationReceived += channel_PushNotificationReceived;
+ ExampleBackgroundTask.Register();
+
if (settings.Values[DEVICE_KEY] == null)
{
Guid uuid = await registerDevice(true);
@@ -134,7 +136,11 @@ namespace Usergrid.Notifications.Client
void channel_PushNotificationReceived(PushNotificationChannel sender, PushNotificationReceivedEventArgs args)
{
- throw new NotImplementedException();
+ var test = "test";
+ if (args.NotificationType == PushNotificationType.Raw)
+ {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index 2120eeb..80b6dd8 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -1,4 +1,5 @@
-/*
+using Newtonsoft.Json.Linq;
+/*
*
* * Licensed to the Apache Software Foundation (ASF) under one or more
* * contributor license agreements. The ASF licenses this file to You
@@ -44,7 +45,7 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "http://server/";
+ serverUrl = "http://yoururl/";
//TODO: change me to your org
org = "test-organization";
//TODO: change me to your app
@@ -52,9 +53,9 @@ namespace Usergrid.Notifications
//TODO: change me to your notifier name
notifier = "windows";
//TODO: change me to your user
- user = "test";
+ user = "mobileuser";
//TODO: change me to your password
- password = "test";
+ password = "password";
this.NavigationCacheMode = NavigationCacheMode.Required;
usergrid = new Client.Usergrid(serverUrl, org, app, user, password, notifier);
@@ -126,7 +127,10 @@ namespace Usergrid.Notifications
Result.Text = "Sending....";
var message = this.pushText.Text;
- if (await usergrid.Push.SendRaw(message))
+
+ JObject obj = new JObject();
+ obj.Add("message", message);
+ if (await usergrid.Push.SendRaw(obj))
{
Result.Text = "It did! :)";
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/MyBackgroundTask.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MyBackgroundTask.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MyBackgroundTask.cs
new file mode 100644
index 0000000..9d8a446
--- /dev/null
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MyBackgroundTask.cs
@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Windows.ApplicationModel.Background;
+
+namespace Usergrid.Notifications
+{
+
+ public sealed class ExampleBackgroundTask : IBackgroundTask
+ {
+ public void Run(IBackgroundTaskInstance taskInstance)
+ {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/Package.appxmanifest
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Package.appxmanifest b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Package.appxmanifest
index 03165e6..2eec1a9 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Package.appxmanifest
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Package.appxmanifest
@@ -45,6 +45,14 @@
<m3:Rotation Preference="portrait" />
</m3:InitialRotationPreference>
</m3:VisualElements>
+ <Extensions>
+ <Extension Category="windows.backgroundTasks" EntryPoint="Usergrid.Notifications.ExampleBackgroundTask">
+ <BackgroundTasks>
+ <Task Type="pushNotification" />
+ </BackgroundTasks>
+ </Extension>
+
+ </Extensions>
</Application>
</Applications>
<Capabilities>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.csproj
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.csproj b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.csproj
index f5ec3fa..b2361e8 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.csproj
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.csproj
@@ -84,6 +84,7 @@
<Compile Include="App.xaml.cs">
<DependentUpon>App.xaml</DependentUpon>
</Compile>
+ <Compile Include="BackgroundTask.cs" />
<Compile Include="Client\EntityResponse.cs" />
<Compile Include="Client\IUsergridClient.cs" />
<Compile Include="Client\PushClient.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.sln
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.sln b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.sln
new file mode 100644
index 0000000..79e96ec
--- /dev/null
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Usergrid.Notifications.sln
@@ -0,0 +1,40 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2013
+VisualStudioVersion = 12.0.31101.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Usergrid.Notifications", "Usergrid.Notifications.csproj", "{EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Debug|ARM = Debug|ARM
+ Debug|x86 = Debug|x86
+ Release|Any CPU = Release|Any CPU
+ Release|ARM = Release|ARM
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|Any CPU.Deploy.0 = Debug|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|ARM.ActiveCfg = Debug|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|ARM.Build.0 = Debug|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|ARM.Deploy.0 = Debug|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|x86.ActiveCfg = Debug|x86
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|x86.Build.0 = Debug|x86
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Debug|x86.Deploy.0 = Debug|x86
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|Any CPU.Build.0 = Release|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|Any CPU.Deploy.0 = Release|Any CPU
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|ARM.ActiveCfg = Release|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|ARM.Build.0 = Release|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|ARM.Deploy.0 = Release|ARM
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|x86.ActiveCfg = Release|x86
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|x86.Build.0 = Release|x86
+ {EB364B8E-B7C1-45C2-BF13-1CBF731EA8B6}.Release|x86.Deploy.0 = Release|x86
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/381d8c57/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
index 3a9c5f2..3afc279 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
@@ -661,7 +661,7 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
+ int size = 10;
int queryLimit = Query.MAX_LIMIT;
int matchMax = queryLimit - 1;
@@ -719,9 +719,9 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 100;
- int startValue = 100;
+ int size = 10;
+ int queryLimit = 5;
+ int startValue = 5;
long start = System.currentTimeMillis();
@@ -779,9 +779,9 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 50;
- int startValue = 100;
+ int size = 20;
+ int queryLimit = 5;
+ int startValue = 10;
long start = System.currentTimeMillis();
@@ -839,9 +839,9 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 100;
- int startValue = 100;
+ int size = 20;
+ int queryLimit = 10;
+ int startValue = 10;
long start = System.currentTimeMillis();
@@ -898,9 +898,9 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 100;
- int startValue = 99;
+ int size = 20;
+ int queryLimit = 10;
+ int startValue = 9;
long start = System.currentTimeMillis();
@@ -957,10 +957,10 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 100;
- int startValue = 50;
- int endValue = 150;
+ int size = 20;
+ int queryLimit = 10;
+ int startValue = 5;
+ int endValue = 15;
long start = System.currentTimeMillis();
@@ -1020,10 +1020,10 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
- int queryLimit = 100;
- int startValue = 50;
- int endValue = 150;
+ int size = 20;
+ int queryLimit = 10;
+ int startValue = 5;
+ int endValue = 15;
long start = System.currentTimeMillis();
@@ -1088,7 +1088,7 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 300;
+ int size = 30;
long start = System.currentTimeMillis();
@@ -1109,7 +1109,7 @@ public class IteratingQueryIT {
Thread.sleep(500);
Query query = new Query();
- query.setLimit( 100 );
+ query.setLimit( 10 );
int count = 0;
@@ -1142,7 +1142,7 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
+ int size = 20;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
@@ -1249,7 +1249,7 @@ public class IteratingQueryIT {
io.doSetup();
- int size = 200;
+ int size = 20;
int queryLimit = Query.MAX_LIMIT;
// the number of entities that should be written including an intersection
@@ -1361,7 +1361,7 @@ public class IteratingQueryIT {
/**
* Leave this as a large size. We have to write over 1k to reproduce this issue
*/
- int size = 200;
+ int size = 20;
long start = System.currentTimeMillis();
@@ -1381,7 +1381,7 @@ public class IteratingQueryIT {
LOG.info( "Writes took {} ms", stop - start );
Query query = Query.fromQL("select * where NOT boolean = false order by index asc");
- query.setLimit( 20 );
+ query.setLimit( 2 );
int index = 0;
int count = 0;
[12/52] [abbrv] incubator-usergrid git commit: check trailing page
Posted by to...@apache.org.
check trailing page
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1f83f0fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1f83f0fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1f83f0fa
Branch: refs/heads/USERGRID-593
Commit: 1f83f0fabcac3281024637300978ff8eb7ec366e
Parents: 0db7f4c
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 16:43:31 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 16:43:31 2015 -0600
----------------------------------------------------------------------
.../query/OrderByDiscardPagesPagingIT.java | 164 -------------------
.../persistence/index/CandidateResults.java | 9 +-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 17 +-
4 files changed, 20 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
deleted file mode 100644
index 2cee402..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.query;
-
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.index.query.Query;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.cassandra.QueryProcessor;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-/**
- *
- */
-public class OrderByDiscardPagesPagingIT extends IteratingQueryIT {
-
- private static final Logger LOG = LoggerFactory.getLogger( OrderByDiscardPagesPagingIT.class );
-
- private static final String search = "select * where field1 = true AND field2 = true order by ordinal";
-
- private static final int PAGE_SIZE = 300;
-
-
- @Test
- public void testUnionPagingCollection() throws Exception {
-
-
- final CollectionIoHelper collectionIoHelper = new CollectionIoHelper( app );
-
- Set<UUID> created = performSetup( collectionIoHelper );
-
-
- testUnionPaging( collectionIoHelper, search, created );
- }
-
-
- @Test
- public void testUnionPagingConnection() throws Exception {
-
- final ConnectionHelper connectionHelper = new ConnectionHelper( app );
-
- Set<UUID> created = performSetup( connectionHelper );
-
-
- testUnionPaging( connectionHelper, search, created );
- }
-
-
- private Set<UUID> performSetup( final IoHelper io ) throws Exception {
- io.doSetup();
-
- int size = ( int ) ( QueryProcessor.PAGE_SIZE * 2.5 );
-
- long start = System.currentTimeMillis();
-
- LOG.info( "Writing {} entities.", size );
-
- Set<UUID> entites = new HashSet<UUID>( size );
-
- for ( int i = 0; i < size; i++ ) {
-
- Map<String, Object> entity = new HashMap<String, Object>();
-
- entity.put( "ordinal", i );
-
- int segment = i / PAGE_SIZE;
-
- boolean shouldBeReturned = segment % 2 != 0;
-
- if ( shouldBeReturned ) {
-
- entity.put( "field1", true );
- entity.put( "field2", true );
- }
- else {
- entity.put( "field1", false );
- entity.put( "field2", false );
- }
-
-
- Entity saved = io.writeEntity( entity );
-
- LOG.info( "Writing entity with id '{}'", saved.getUuid() );
-
- if ( shouldBeReturned ) {
- entites.add( saved.getUuid() );
- }
- }
-
- long stop = System.currentTimeMillis();
-
- LOG.info( "Writes took {} ms", stop - start );
-
- return entites ;
- }
-
-
- private void testUnionPaging( final IoHelper io, final String queryString, final Set<UUID> expectedResults )
- throws Exception {
-
- //our field1Or has a result size < our page size, so it shouldn't blow up when the cursor is getting created
- //the leaf iterator should insert it's own "no value left" into the cursor
- Query query = Query.fromQL( queryString );
- query.setLimit( PAGE_SIZE );
-
- Results results;
-
- long start = System.currentTimeMillis();
-
- do {
-
- // now do simple ordering, should be returned in order
- results = io.getResults( query );
-
- final List<Entity> entities = results.getEntities();
-
- for ( int i = 0; i < entities.size(); i++ ) {
- final UUID uuid = entities.get( i ).getUuid();
-
- assertTrue( "Value should not be returned twice", expectedResults.contains( uuid ) );
-
- expectedResults.remove( uuid );
- }
-
- query.setCursor( results.getCursor() );
- }
- while ( results.getCursor() != null );
-
- long stop = System.currentTimeMillis();
-
- LOG.info( "Query took {} ms to return {} entities", stop - start, expectedResults.size() );
-
- assertEquals( "All entities returned", 0, expectedResults.size() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index f0955ea..14130bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -49,12 +49,16 @@ public class CandidateResults implements Iterable<CandidateResult> {
private final List<CandidateResult> candidates;
private final Collection<SelectFieldMapping> getFieldMappings;
+ private final String esCursor;
public CandidateResults( List<CandidateResult> candidates,
- final Collection<SelectFieldMapping> getFieldMappings ) {
+ final Collection<SelectFieldMapping> getFieldMappings,
+ final String esCursor
+ ) {
this.candidates = candidates;
this.getFieldMappings = getFieldMappings;
+ this.esCursor = esCursor;
}
@@ -109,4 +113,7 @@ public class CandidateResults implements Iterable<CandidateResult> {
return candidates.iterator();
}
+ public String getEsCursor() {
+ return esCursor;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 6574985..5b05b27 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -310,8 +310,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
candidates.add( candidateResult );
}
- final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings());
final String esScrollCursor = searchResponse.getScrollId();
+ final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor);
// >= seems odd. However if we get an overflow, we need to account for it.
if ( esScrollCursor != null && length >= limit ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index e95c2bf..9b8a9f5 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -641,6 +641,7 @@ public class EntityIndexTest extends BaseIT {
@Test
public void testCursorFormat() throws Exception {
+ String myType = UUID.randomUUID().toString();
Id appId = new SimpleId( "application" );
Id ownerId = new SimpleId( "owner" );
@@ -655,7 +656,7 @@ public class EntityIndexTest extends BaseIT {
final EntityIndexBatch batch = entityIndex.createBatch();
- final int size = 10;
+ final int size = 100;
final List<Id> entityIds = new ArrayList<>( size );
@@ -671,6 +672,7 @@ public class EntityIndexTest extends BaseIT {
put( "email", "ed@anuff.com" );
put( "middlename", middleName );
put( "ordinal", ordinal );
+ put( "mytype", myType);
}};
final Id userId = new SimpleId( "user" );
@@ -691,17 +693,18 @@ public class EntityIndexTest extends BaseIT {
ei.refreshAsync().toBlocking().last();
- final int limit = 1;
+ final int limit = 5;
final int expectedPages = size / limit;
String cursor = null;
+ UUID lastId = null;
for ( int i = 0; i < expectedPages; i++ ) {
//**
- final String query = "select * order by ordinal asc";
+ final String query = "select * where mytype='"+myType+"' order by ordinal asc";
final CandidateResults results =
cursor == null ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit ) :
@@ -713,10 +716,12 @@ public class EntityIndexTest extends BaseIT {
assertEquals( "Should be 16 bytes as hex", 32, cursor.length() );
- assertEquals( 1, results.size() );
+ assertEquals(limit, results.size());
-
- assertEquals( results.get( 0 ).getId(), entityIds.get( i ) );
+ int ordinal = 0;//i == 0 ? 0 : 1;
+ assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
+ lastId = results.get((i+1)*limit-1).getId().getUuid();
+ assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) );
}
//get our next page, we shouldn't get a cursor
[52/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' into USERGRID-593
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-593
# Conflicts:
# stack/core/pom.xml
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
# stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
# stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
# stack/corepersistence/pom.xml
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
# stack/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ec3a710a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ec3a710a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ec3a710a
Branch: refs/heads/USERGRID-593
Commit: ec3a710adf5fd8c88e27e3c248d50c45c7164d18
Parents: 6904c9d 852c974
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 09:59:57 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 09:59:57 2015 -0600
----------------------------------------------------------------------
portal/index-template.html | 2 +-
portal/js/data/data-controller.js | 58 +-
portal/js/data/data.html | 36 +-
portal/js/global/ug-service.js | 30 +-
portal/js/login/login-controller.js | 15 +-
.../Usergrid.Notifications/BackgroundTask.cs | 57 +
.../Usergrid.Notifications/Client/PushClient.cs | 12 +-
.../Usergrid.Notifications/MainPage.xaml.cs | 12 +-
.../Usergrid.Notifications/MyBackgroundTask.cs | 18 +
.../Usergrid.Notifications/Package.appxmanifest | 8 +
.../Usergrid.Notifications.csproj | 1 +
.../Usergrid.Notifications.sln | 40 +
.../main/dist/init_instance/init_rest_server.sh | 6 +-
.../main/resources/usergrid-default.properties | 5 +-
stack/core/pom.xml | 46 +-
.../batch/service/JobSchedulerService.java | 28 +-
.../batch/service/SchedulerService.java | 2 +-
.../batch/service/SchedulerServiceImpl.java | 23 +-
.../corepersistence/ApplicationIdCache.java | 51 +
.../ApplicationIdCacheFactory.java | 43 +
.../corepersistence/ApplicationIdCacheFig.java | 43 +
.../corepersistence/ApplicationIdCacheImpl.java | 146 +
.../usergrid/corepersistence/CoreModule.java | 137 +-
.../corepersistence/CpEntityManager.java | 411 +--
.../corepersistence/CpEntityManagerFactory.java | 579 ++--
.../corepersistence/CpManagerCache.java | 65 +-
.../corepersistence/CpQueryProcessor.java | 94 -
.../corepersistence/CpRelationManager.java | 1842 ++---------
.../usergrid/corepersistence/CpSetup.java | 32 +-
.../usergrid/corepersistence/CpWalker.java | 83 +-
.../usergrid/corepersistence/GuiceFactory.java | 75 +-
.../usergrid/corepersistence/ManagerCache.java | 17 +-
.../corepersistence/OrgApplicationCache.java | 67 -
.../OrgApplicationCacheImpl.java | 181 --
.../events/EntityDeletedHandler.java | 43 +-
.../events/EntityVersionCreatedHandler.java | 22 +-
.../events/EntityVersionDeletedHandler.java | 144 +-
.../index/AsyncIndexProvider.java | 119 +
.../index/AsyncIndexService.java | 42 +
.../index/InMemoryAsyncIndexService.java | 88 +
.../corepersistence/index/IndexEntityEvent.java | 75 +
.../index/IndexProcessorFig.java | 86 +
.../corepersistence/index/IndexService.java | 52 +
.../corepersistence/index/IndexServiceImpl.java | 172 +
.../corepersistence/index/ReIndexAction.java | 33 +
.../corepersistence/index/ReIndexService.java | 89 +
.../index/ReIndexServiceImpl.java | 129 +
.../index/SQSAsyncIndexService.java | 325 ++
.../migration/AppInfoMigrationPlugin.java | 291 ++
.../migration/AppInfoVersions.java | 40 +
.../migration/CoreDataVersions.java | 45 +
.../migration/CoreMigration.java | 53 +
.../migration/CoreMigrationPlugin.java | 64 +
.../migration/EntityDataMigration.java | 147 -
.../migration/EntityTypeMappingMigration.java | 69 +-
.../migration/GraphShardVersionMigration.java | 152 -
.../migration/MigrationModuleVersionPlugin.java | 145 +
.../migration/MigrationSystemVersions.java | 43 +
.../corepersistence/migration/Versions.java | 14 -
.../CollectionResultsLoaderFactoryImpl.java | 6 +-
.../ConnectionResultsLoaderFactoryImpl.java | 6 +-
.../results/ElasticSearchQueryExecutor.java | 92 +-
.../corepersistence/results/EntityVerifier.java | 2 +-
.../results/FilteringLoader.java | 122 +-
.../corepersistence/results/ResultsLoader.java | 2 +-
.../results/ResultsLoaderFactory.java | 7 +-
.../results/ResultsVerifier.java | 4 +-
.../results/VersionVerifier.java | 12 +-
.../rx/AllEntitiesInSystemObservable.java | 101 -
.../rx/ApplicationObservable.java | 130 -
.../rx/EdgesFromSourceObservable.java | 65 -
.../rx/EdgesToTargetObservable.java | 65 -
.../corepersistence/rx/TargetIdObservable.java | 66 -
.../rx/impl/AbstractGraphVisitorImpl.java | 65 +
.../rx/impl/AllApplicationsObservable.java | 43 +
.../rx/impl/AllApplicationsObservableImpl.java | 133 +
.../rx/impl/AllEntitiesInSystemImpl.java | 53 +
.../rx/impl/AllEntityIdsObservable.java | 52 +
.../rx/impl/AllEntityIdsObservableImpl.java | 92 +
.../rx/impl/AllNodesInGraphImpl.java | 54 +
.../corepersistence/rx/impl/EdgeScope.java | 51 +
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 247 +-
.../util/SerializableMapper.java | 91 +
.../usergrid/exception/ConflictException.java | 36 +
.../main/java/org/apache/usergrid/mq/Query.java | 2 +-
.../org/apache/usergrid/mq/QueryProcessor.java | 210 +-
.../apache/usergrid/mq/cassandra/QueuesCF.java | 5 -
.../usergrid/persistence/CounterQuery.java | 2 +-
.../usergrid/persistence/EntityManager.java | 51 +-
.../persistence/EntityManagerFactory.java | 134 +-
.../persistence/EntityPropertyComparator.java | 56 -
.../persistence/GuiceAdapterBeanFactory.java | 114 +
.../persistence/MultiQueryIterator.java | 4 +-
.../persistence/ObservableIterator.java | 1 +
.../persistence/PagingResultsIterator.java | 4 +-
.../apache/usergrid/persistence/PathQuery.java | 6 +-
.../usergrid/persistence/PersistenceModule.java | 66 +
.../org/apache/usergrid/persistence/Query.java | 912 ++++++
.../apache/usergrid/persistence/QueryUtils.java | 61 +-
.../usergrid/persistence/RelationManager.java | 7 +-
.../apache/usergrid/persistence/Results.java | 70 +-
.../org/apache/usergrid/persistence/Schema.java | 5 +-
.../persistence/cassandra/ApplicationCF.java | 37 -
.../cassandra/CassandraPersistenceUtils.java | 122 +-
.../persistence/cassandra/CassandraService.java | 304 +-
.../persistence/cassandra/CursorCache.java | 137 -
.../cassandra/EntityManagerFactoryImpl.java | 485 ---
.../cassandra/EntityManagerImpl.java | 2946 ------------------
.../cassandra/EntityValueSerializer.java | 53 -
.../persistence/cassandra/GeoIndexManager.java | 330 --
.../persistence/cassandra/IndexUpdate.java | 451 ---
.../persistence/cassandra/QueryProcessor.java | 60 -
.../cassandra/QueryProcessorImpl.java | 727 -----
.../cassandra/RelationManagerImpl.java | 2298 --------------
.../persistence/cassandra/SetupImpl.java | 170 -
.../cassandra/SimpleIndexBucketLocatorImpl.java | 122 -
.../cassandra/index/ConnectedIndexScanner.java | 280 --
.../cassandra/index/IndexBucketScanner.java | 238 --
.../index/IndexMultiBucketSetLoader.java | 139 -
.../cassandra/index/IndexScanner.java | 40 -
.../cassandra/index/NoOpIndexScanner.java | 95 -
.../DuplicateUniquePropertyExistsException.java | 2 +-
.../persistence/geo/CollectionGeoSearch.java | 68 -
.../persistence/geo/ConnectionGeoSearch.java | 67 -
.../persistence/geo/EntityLocationRef.java | 227 --
.../EntityLocationRefDistanceComparator.java | 78 -
.../persistence/geo/GeoIndexSearcher.java | 370 ---
.../persistence/geo/GeocellManager.java | 195 --
.../usergrid/persistence/geo/GeocellUtils.java | 543 ----
.../geo/comparator/DoubleTupleComparator.java | 39 -
.../persistence/geo/model/BoundingBox.java | 74 -
.../persistence/geo/model/CostFunction.java | 36 -
.../geo/model/DefaultCostFunction.java | 37 -
.../usergrid/persistence/geo/model/Point.java | 61 -
.../usergrid/persistence/geo/model/Tuple.java | 40 -
.../usergrid/persistence/query/ir/AllNode.java | 82 -
.../usergrid/persistence/query/ir/AndNode.java | 44 -
.../persistence/query/ir/BooleanNode.java | 65 -
.../query/ir/EmailIdentifierNode.java | 58 -
.../query/ir/NameIdentifierNode.java | 56 -
.../persistence/query/ir/NodeVisitor.java | 79 -
.../usergrid/persistence/query/ir/NotNode.java | 75 -
.../usergrid/persistence/query/ir/OrNode.java | 53 -
.../persistence/query/ir/OrderByNode.java | 105 -
.../persistence/query/ir/QueryNode.java | 41 -
.../persistence/query/ir/QuerySlice.java | 345 --
.../persistence/query/ir/SearchVisitor.java | 270 --
.../persistence/query/ir/SliceNode.java | 180 --
.../query/ir/UuidIdentifierNode.java | 60 -
.../persistence/query/ir/WithinNode.java | 109 -
.../query/ir/result/AbstractScanColumn.java | 83 -
.../result/CollectionResultsLoaderFactory.java | 41 -
.../ir/result/ConnectionIndexSliceParser.java | 87 -
.../query/ir/result/ConnectionRefLoader.java | 81 -
.../result/ConnectionResultsLoaderFactory.java | 50 -
.../ir/result/ConnectionTypesIterator.java | 190 --
.../query/ir/result/EmptyIterator.java | 63 -
.../query/ir/result/EntityRefLoader.java | 51 -
.../query/ir/result/EntityResultsLoader.java | 47 -
.../query/ir/result/GeoIterator.java | 351 ---
.../persistence/query/ir/result/IDLoader.java | 40 -
.../query/ir/result/IntersectionIterator.java | 170 -
.../query/ir/result/MergeIterator.java | 150 -
.../query/ir/result/MultiIterator.java | 55 -
.../query/ir/result/OrderByIterator.java | 250 --
.../query/ir/result/ResultIterator.java | 44 -
.../query/ir/result/ResultsLoader.java | 30 -
.../query/ir/result/ResultsLoaderFactory.java | 37 -
.../persistence/query/ir/result/ScanColumn.java | 32 -
.../query/ir/result/ScanColumnTransformer.java | 39 -
.../ir/result/SecondaryIndexSliceParser.java | 61 -
.../query/ir/result/SliceIterator.java | 242 --
.../query/ir/result/SliceParser.java | 32 -
.../query/ir/result/StaticIdIterator.java | 82 -
.../query/ir/result/SubtractionIterator.java | 113 -
.../query/ir/result/UUIDIndexSliceParser.java | 47 -
.../query/ir/result/UnionIterator.java | 264 --
.../main/resources/usergrid-core-context.xml | 32 +-
.../java/org/apache/usergrid/Application.java | 2 +-
.../org/apache/usergrid/CoreApplication.java | 37 +-
.../java/org/apache/usergrid/CoreITSetup.java | 10 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 69 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../usergrid/batch/job/SchedulerRuntime8IT.java | 8 +-
.../corepersistence/StaleIndexCleanupTest.java | 111 +-
.../corepersistence/TestIndexModule.java | 43 +
.../index/AsyncIndexServiceTest.java | 203 ++
.../index/InMemoryAsycIndexServiceTest.java | 64 +
.../corepersistence/index/IndexServiceTest.java | 327 ++
.../corepersistence/index/PublishRxTest.java | 95 +
.../index/SQSAsyncIndexServiceTest.java | 108 +
.../migration/EntityDataMigrationIT.java | 262 --
.../migration/EntityTypeMappingMigrationIT.java | 183 +-
.../migration/GraphShardVersionMigrationIT.java | 226 --
.../MigrationModuleVersionPluginTest.java | 259 ++
.../migration/MigrationTestRule.java | 99 -
.../migration/TestProgressObserver.java | 71 -
.../rx/AllEntitiesInSystemObservableIT.java | 71 +-
.../rx/ApplicationObservableTestIT.java | 17 +-
.../rx/EdgesFromSourceObservableIT.java | 19 +-
.../rx/EdgesToTargetObservableIT.java | 59 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../usergrid/persistence/CollectionIT.java | 379 ++-
.../apache/usergrid/persistence/CounterIT.java | 11 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 +-
.../persistence/EntityConnectionsIT.java | 16 +-
.../usergrid/persistence/EntityManagerIT.java | 62 +-
.../EntityPropertyComparatorTest.java | 222 --
.../org/apache/usergrid/persistence/GeoIT.java | 122 +-
.../persistence/GeoQueryBooleanTest.java | 24 +-
.../apache/usergrid/persistence/IndexIT.java | 64 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../usergrid/persistence/PathQueryIT.java | 27 +-
.../PerformanceEntityRebuildIndexTest.java | 60 +-
.../usergrid/persistence/PermissionsIT.java | 6 +-
.../apache/usergrid/persistence/QueryTest.java | 328 --
.../cassandra/EntityManagerFactoryImplIT.java | 126 +-
.../cassandra/QueryProcessorTest.java | 822 -----
.../SimpleIndexShardLocatorImplTest.java | 177 --
...EntityLocationRefDistanceComparatorTest.java | 104 -
.../persistence/query/CollectionIoHelper.java | 4 +-
.../persistence/query/ConnectionHelper.java | 4 +-
.../query/IntersectionTransitivePagingIT.java | 6 +-
.../query/IntersectionUnionPagingIT.java | 6 +-
.../usergrid/persistence/query/IoHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 157 +-
.../persistence/query/NotSubPropertyIT.java | 4 +-
.../query/OrderByDiscardPagesPagingIT.java | 164 -
.../query/ir/result/AbstractScanColumnTest.java | 101 -
.../query/ir/result/InOrderIterator.java | 132 -
.../ir/result/IntersectionIteratorTest.java | 308 --
.../query/ir/result/IteratorHelper.java | 34 -
.../ir/result/SubtractionIteratorTest.java | 182 --
.../query/ir/result/UnionIteratorTest.java | 468 ---
.../apache/usergrid/utils/EdgeTestUtils.java | 50 +
stack/core/src/test/resources/log4j.properties | 4 +
.../resources/usergrid-custom-test.properties | 4 +-
.../resources/usergrid-properties-context.xml | 40 -
.../test/resources/usergrid-test-context.xml | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../persistence/collection/CollectionScope.java | 45 -
.../collection/EntityCollectionManager.java | 55 +-
.../EntityCollectionManagerFactory.java | 36 +-
.../collection/EntityCollectionManagerSync.java | 49 -
.../persistence/collection/FieldSet.java | 2 +-
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 21 +-
.../collection/event/EntityDeleted.java | 8 +-
.../collection/event/EntityVersionCreated.java | 7 +-
.../collection/event/EntityVersionDeleted.java | 11 +-
.../exception/CollectionRuntimeException.java | 30 +-
.../exception/EntityTooLargeException.java | 6 +-
.../exception/WriteCommitException.java | 11 +-
.../WriteOptimisticVerifyException.java | 11 +-
.../exception/WriteStartException.java | 11 +-
.../exception/WriteUniqueVerifyException.java | 6 +-
.../collection/guice/CollectionModule.java | 48 +-
.../guice/CollectionTaskExecutor.java | 4 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../collection/impl/CollectionScopeImpl.java | 102 -
.../EntityCollectionManagerFactoryImpl.java | 93 +-
.../impl/EntityCollectionManagerImpl.java | 197 +-
.../impl/EntityCollectionManagerSyncImpl.java | 71 -
.../collection/impl/EntityDeletedTask.java | 58 +-
.../impl/EntityVersionCleanupTask.java | 197 +-
.../impl/EntityVersionCreatedTask.java | 39 +-
.../impl/EntityVersionTaskFactory.java | 17 +-
.../mvcc/MvccEntitySerializationStrategy.java | 102 -
.../mvcc/MvccLogEntrySerializationStrategy.java | 80 -
.../collection/mvcc/changelog/ChangeLog.java | 2 -
.../mvcc/entity/MvccValidationUtils.java | 16 -
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 8 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 11 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 6 +-
.../mvcc/stage/CollectionIoEvent.java | 9 +-
.../mvcc/stage/EntityUpdateEvent.java | 34 -
.../mvcc/stage/delete/MarkCommit.java | 15 +-
.../collection/mvcc/stage/delete/MarkStart.java | 20 +-
.../mvcc/stage/write/RollbackAction.java | 14 +-
.../mvcc/stage/write/WriteCommit.java | 32 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 12 +-
.../collection/mvcc/stage/write/WriteStart.java | 22 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 45 +-
.../collection/serialization/EntityRepair.java | 38 -
.../MvccEntitySerializationStrategy.java | 117 +
.../MvccLogEntrySerializationStrategy.java | 82 +
.../serialization/OptimisticUpdate.java | 23 -
.../collection/serialization/UniqueValue.java | 1 -
.../UniqueValueSerializationStrategy.java | 65 +-
.../serialization/UniqueValueSet.java | 1 +
.../impl/CollectionDataVersions.java | 54 +
.../serialization/impl/EntityRepairImpl.java | 149 -
.../impl/EntityVersionSerializer.java | 11 +-
.../serialization/impl/FieldSerializer.java | 118 -
.../serialization/impl/LogEntryIterator.java | 8 +-
.../serialization/impl/MutableFieldSet.java | 6 +-
.../MvccEntitySerializationStrategyImpl.java | 177 +-
...vccEntitySerializationStrategyProxyImpl.java | 163 +-
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 574 ++++
.../MvccLogEntrySerializationProxyImpl.java | 149 +
.../MvccLogEntrySerializationStrategyImpl.java | 92 +-
...MvccLogEntrySerializationStrategyV1Impl.java | 118 +
...MvccLogEntrySerializationStrategyV2Impl.java | 106 +
.../serialization/impl/SerializationModule.java | 170 +-
.../serialization/impl/TypeField.java | 43 +
.../serialization/impl/UniqueFieldEntry.java | 60 +
.../impl/UniqueFieldEntrySerializer.java | 138 +
.../impl/UniqueFieldRowKeySerializer.java | 122 +
.../impl/UniqueTypeFieldRowKeySerializer.java | 60 +
.../serialization/impl/UniqueValueImpl.java | 5 +-
.../UniqueValueSerializationStrategyImpl.java | 322 +-
...iqueValueSerializationStrategyProxyImpl.java | 179 ++
.../UniqueValueSerializationStrategyV1Impl.java | 159 +
.../UniqueValueSerializationStrategyV2Impl.java | 139 +
.../impl/migration/CollectionMigration.java | 53 +
.../migration/CollectionMigrationPlugin.java | 68 +
.../impl/migration/EntityIdScope.java | 79 +
.../migration/MvccEntityDataMigrationImpl.java | 280 ++
.../serialization/impl/util/Inflector.java | 370 +++
.../impl/util/LegacyScopeUtils.java | 51 +
.../collection/util/EntityUtils.java | 71 -
.../collection/ApplicationContextImplTest.java | 33 +
.../collection/CollectionContextImplTest.java | 53 -
.../EntityCollectionManagerFactoryTest.java | 9 +-
.../collection/EntityCollectionManagerIT.java | 271 +-
.../EntityCollectionManagerStressTest.java | 14 +-
.../EntityCollectionManagerSyncIT.java | 193 --
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 683 ++--
.../impl/EntityVersionCreatedTaskTest.java | 29 +-
.../mvcc/stage/AbstractEntityStageTest.java | 6 +-
.../mvcc/stage/AbstractIdStageTest.java | 4 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 6 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 14 +-
.../mvcc/stage/delete/MarkStartTest.java | 6 +-
.../mvcc/stage/write/FieldSerializerTest.java | 52 -
.../write/UniqueFieldRowKeySerializerTest.java | 52 +
...niqueValueSerializationStrategyImplTest.java | 185 --
.../mvcc/stage/write/WriteCommitTest.java | 12 +-
.../stage/write/WriteOptimisticVerifyTest.java | 35 +-
.../mvcc/stage/write/WriteStartTest.java | 26 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 34 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 13 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
.../serialization/EntityRepairImplTest.java | 147 -
.../impl/LogEntryIteratorTest.java | 17 +-
...MvccEntitySerializationStrategyImplTest.java | 165 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 -
...ntitySerializationStrategyProxyV1_3Test.java | 81 +
...cEntitySerializationStrategyProxyV2Test.java | 83 -
...ntitySerializationStrategyProxyV2_3Test.java | 82 +
...ccEntitySerializationStrategyV1ImplTest.java | 78 +-
...ccEntitySerializationStrategyV2ImplTest.java | 73 +-
.../MvccEntitySerializationStrategyV2Test.java | 27 +-
...ccEntitySerializationStrategyV3ImplTest.java | 70 +
.../impl/MvccLESSTransientTest.java | 27 +-
...ccLogEntrySerializationStrategyImplTest.java | 107 +-
...EntrySerializationStrategyProxyImplTest.java | 90 +
...LogEntrySerializationStrategyV1ImplTest.java | 55 +
...LogEntrySerializationStrategyV2ImplTest.java | 53 +
.../impl/ScopedRowKeySerializerTest.java | 2 +-
.../impl/SerializationComparison.java | 4 +-
.../impl/SettingsValidationTest.java | 1 -
.../impl/UniqueFieldEntrySerializerTest.java | 108 +
...niqueValueSerializationStrategyImplTest.java | 344 ++
...ValueSerializationStrategyProxyImplTest.java | 100 +
...queValueSerializationStrategyV1ImplTest.java | 57 +
...queValueSerializationStrategyV2ImplTest.java | 58 +
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 214 ++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 99 +
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +
.../collection/util/EntityHelper.java | 1 -
.../collection/util/InvalidEntityGenerator.java | 1 +
.../collection/util/LogEntryMock.java | 47 +-
.../collection/util/UniqueValueEntryMock.java | 150 +
.../collection/util/VersionGenerator.java | 55 +
stack/corepersistence/common/pom.xml | 21 +-
.../usergrid/persistence/core/CPManager.java | 28 +
.../persistence/core/astyanax/CassandraFig.java | 3 -
.../astyanax/DynamicCompositeParserImpl.java | 81 +
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../persistence/core/guice/CommonModule.java | 27 +-
.../persistence/core/guice/CurrentImpl.java | 42 -
.../persistence/core/guice/PreviousImpl.java | 42 -
.../persistence/core/guice/ProxyImpl.java | 42 -
.../core/hystrix/HystrixCassandra.java | 94 -
.../core/metrics/MetricsFactory.java | 2 +-
.../core/metrics/MetricsFactoryImpl.java | 44 +-
.../core/metrics/ObservableTimer.java | 72 +
.../migration/data/AbstractMigrationPlugin.java | 128 +
.../core/migration/data/DataMigration.java | 97 +-
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 237 +-
.../migration/data/MigrationDataProvider.java | 43 +
.../core/migration/data/MigrationInfoCache.java | 52 +
.../migration/data/MigrationInfoCacheImpl.java | 84 +
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 +
.../migration/data/MigrationRelationship.java | 117 +
.../core/migration/data/PluginPhase.java | 42 +
.../core/migration/data/ProgressObserver.java | 63 +
.../core/migration/data/VersionedData.java | 38 +
.../migration/data/VersionedMigrationSet.java | 164 +
.../core/migration/schema/Migration.java | 2 +-
.../persistence/core/rx/ObservableIterator.java | 2 +
.../persistence/core/rx/RxSchedulerFig.java | 60 +
.../persistence/core/rx/RxTaskScheduler.java | 40 +
.../core/rx/RxTaskSchedulerImpl.java | 131 +
.../core/scope/ApplicationScope.java | 9 +-
.../core/scope/ApplicationScopeImpl.java | 24 +-
.../persistence/core/task/TaskExecutor.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 +
.../MultiKeyColumnNameIteratorTest.java | 187 +-
.../astyanax/MultiRowColumnIteratorTest.java | 50 +-
.../persistence/core/aws/NoAWSCredsRule.java | 98 +
.../core/guice/DataMigrationResetRule.java | 88 +
.../core/guice/MaxMigrationModule.java | 39 -
.../core/guice/MaxMigrationVersion.java | 40 -
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++-
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 +
.../migration/data/TestProgressObserver.java | 89 +
.../data/VersionedMigrationSetTest.java | 194 ++
.../persistence/core/util/IdGenerator.java | 51 +
.../persistence/graph/GraphManager.java | 3 +-
.../persistence/graph/GraphManagerFactory.java | 2 +
.../persistence/graph/guice/GraphModule.java | 106 +-
.../graph/impl/GraphManagerImpl.java | 41 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 21 +-
.../impl/stage/NodeDeleteListenerImpl.java | 21 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 71 +
.../graph/serialization/TargetIdObservable.java | 38 +
.../EdgeMetadataSerializationProxyImpl.java | 161 +-
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 129 +
.../serialization/impl/GraphDataVersions.java | 43 +
.../impl/GraphManagerFactoryImpl.java | 95 +
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 +
.../impl/migration/EdgeDataMigrationImpl.java | 136 +
.../impl/migration/GraphMigration.java | 53 +
.../impl/migration/GraphMigrationPlugin.java | 69 +
.../serialization/impl/migration/GraphNode.java | 39 +
.../shard/count/NodeShardApproximationImpl.java | 13 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 64 +-
.../persistence/graph/GraphManagerIT.java | 35 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 12 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 12 +-
.../graph/impl/NodeDeleteListenerTest.java | 10 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 30 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 15 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 23 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 ++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
.../usergrid/persistence/map/MapManager.java | 25 +-
.../persistence/map/guice/MapModule.java | 8 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 80 +
.../model/entity/EntityToMapConverter.java | 160 +
.../model/entity/MapToEntityConverter.java | 203 ++
.../persistence/model/entity/SimpleId.java | 36 +-
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 8 +-
.../persistence/model/field/StringField.java | 20 +
.../persistence/model/util/EntityUtils.java | 72 +
stack/corepersistence/pom.xml | 36 +-
stack/corepersistence/queryindex/pom.xml | 63 +-
.../index/query/tree/CpQueryFilter.g | 20 +-
.../persistence/index/AliasedEntityIndex.java | 13 +-
.../index/ApplicationEntityIndex.java | 55 +
.../persistence/index/CandidateResult.java | 84 +
.../persistence/index/CandidateResults.java | 106 +
.../usergrid/persistence/index/EntityIndex.java | 75 +-
.../persistence/index/EntityIndexBatch.java | 25 +-
.../persistence/index/EntityIndexFactory.java | 11 +-
.../persistence/index/IndexBatchBuffer.java | 36 -
.../persistence/index/IndexBufferConsumer.java | 37 -
.../persistence/index/IndexBufferProducer.java | 37 -
.../usergrid/persistence/index/IndexEdge.java | 36 +
.../usergrid/persistence/index/IndexFig.java | 128 +-
.../persistence/index/IndexIdentifier.java | 92 -
.../index/IndexOperationMessage.java | 138 -
.../persistence/index/IndexRefreshCommand.java | 50 +
.../usergrid/persistence/index/IndexScope.java | 38 -
.../usergrid/persistence/index/SearchEdge.java | 61 +
.../usergrid/persistence/index/SearchType.java | 51 +
.../usergrid/persistence/index/SearchTypes.java | 46 +-
.../persistence/index/SelectFieldMapping.java | 58 +
.../persistence/index/guice/IndexModule.java | 44 +-
.../persistence/index/guice/QueueProvider.java | 116 -
.../persistence/index/impl/BatchOperation.java | 41 +
.../persistence/index/impl/BatchRequest.java | 41 -
.../persistence/index/impl/BufferQueue.java | 68 -
.../index/impl/BufferQueueInMemoryImpl.java | 116 -
.../index/impl/BufferQueueSQSImpl.java | 307 --
.../index/impl/DeIndexOperation.java | 104 +
.../persistence/index/impl/DeIndexRequest.java | 115 -
.../persistence/index/impl/EntityField.java | 148 +
.../index/impl/EntityMappingParser.java | 247 ++
.../index/impl/EntityToMapConverter.java | 108 +
.../impl/EsApplicationEntityIndexImpl.java | 274 ++
.../index/impl/EsEntityIndexBatchImpl.java | 283 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 44 +-
.../index/impl/EsEntityIndexImpl.java | 590 +---
.../index/impl/EsIndexBufferConsumerImpl.java | 348 +--
.../index/impl/EsIndexBufferProducerImpl.java | 65 -
.../persistence/index/impl/EsIndexCache.java | 142 -
.../index/impl/EsIndexCacheImpl.java | 141 +
.../persistence/index/impl/EsProvider.java | 129 +-
.../persistence/index/impl/EsQueryVistor.java | 589 ++--
.../persistence/index/impl/FieldParser.java | 45 +
.../index/impl/FlushBufferQueue.java | 23 +
.../persistence/index/impl/GeoSortFields.java | 95 +
.../persistence/index/impl/IndexAlias.java | 42 +
.../index/impl/IndexBufferConsumer.java | 38 +
.../persistence/index/impl/IndexCache.java | 43 +
.../persistence/index/impl/IndexEdgeImpl.java | 84 +
.../persistence/index/impl/IndexIdentifier.java | 46 +
.../index/impl/IndexIdentifierImpl.java | 67 +
.../persistence/index/impl/IndexOperation.java | 107 +
.../index/impl/IndexOperationMessage.java | 126 +
.../index/impl/IndexRefreshCommandImpl.java | 170 +
.../persistence/index/impl/IndexRequest.java | 125 -
.../persistence/index/impl/IndexScopeImpl.java | 88 -
.../persistence/index/impl/IndexingUtils.java | 267 +-
.../index/impl/NoOpFilterBuilder.java | 45 +
.../index/impl/NoOpQueryBuilder.java | 45 +
.../persistence/index/impl/SearchEdgeImpl.java | 98 +
.../impl/SearchRequestBuilderStrategy.java | 290 ++
.../persistence/index/impl/SortBuilder.java | 44 +
.../index/migration/EsIndexMigrationPlugin.java | 46 +
.../index/migration/IndexDataVersions.java | 39 +
.../index/migration/IndexMigration.java | 35 +
.../index/migration/LegacyIndexIdentifier.java | 78 +
.../index/query/CandidateResult.java | 74 -
.../index/query/CandidateResults.java | 90 -
.../persistence/index/query/EntityResults.java | 108 -
.../persistence/index/query/ParsedQuery.java | 205 ++
.../index/query/ParsedQueryBuilder.java | 108 +
.../usergrid/persistence/index/query/Query.java | 1436 ---------
.../persistence/index/query/Results.java | 148 -
.../persistence/index/query/SortPredicate.java | 144 +
.../index/query/tree/QueryVisitor.java | 55 +-
.../index/utils/IndexValidationUtils.java | 24 +-
.../persistence/index/utils/ListUtils.java | 1 +
.../persistence/index/utils/StringUtils.java | 62 -
.../persistence/index/usergrid-mappings.json | 93 +
.../persistence/index/guice/IndexTestFig.java | 62 +
.../index/guice/TestIndexModule.java | 42 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 -
.../index/impl/CorePerformanceIT.java | 339 --
.../index/impl/ElasticSearchRule.java | 58 +
.../impl/EntityConnectionIndexImplTest.java | 306 --
.../index/impl/EntityIndexMapUtils.java | 52 +-
.../persistence/index/impl/EntityIndexTest.java | 1073 +++++--
.../index/impl/EntityToMapConverterTest.java | 653 ++++
.../persistence/index/impl/GeoPagingTest.java | 163 +
.../index/impl/IndexLoadTestsIT.java | 393 +++
.../persistence/query/tree/GrammarTreeTest.java | 157 +-
.../src/test/resources/dynamic-test.properties | 2 +-
.../src/test/resources/log4j.properties | 10 +-
.../src/test/resources/usergrid-UNIT.properties | 3 +
.../usergrid/persistence/queue/QueueFig.java | 8 +
.../persistence/queue/NoAWSCredsRule.java | 98 -
.../persistence/queue/QueueManagerTest.java | 3 +-
.../queue/guice/TestQueueModule.java | 3 +-
stack/java-sdk-old/pom.xml | 102 -
.../usergrid/mongo/protocol/OpDelete.java | 4 +-
.../apache/usergrid/mongo/protocol/OpQuery.java | 6 +-
.../usergrid/mongo/protocol/OpUpdate.java | 4 +-
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +-
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +-
stack/pom.xml | 28 +-
stack/rest/pom.xml | 17 +-
.../usergrid/rest/AbstractContextResource.java | 102 +-
.../org/apache/usergrid/rest/ApiResponse.java | 50 +-
.../org/apache/usergrid/rest/IndexResource.java | 9 +-
.../apache/usergrid/rest/MigrateResource.java | 122 +-
.../org/apache/usergrid/rest/RootResource.java | 19 +-
.../rest/applications/ApplicationResource.java | 38 +-
.../rest/applications/ServiceResource.java | 43 +-
.../notifiers/NotifiersResource.java | 11 +-
.../rest/applications/users/UsersResource.java | 4 +-
.../exceptions/AbstractExceptionMapper.java | 11 +-
.../exceptions/ConflictExceptionMapper.java | 35 +
.../exceptions/NotFoundExceptionMapper.java | 35 +
.../rest/exceptions/UncaughtException.java | 36 +
.../rest/management/ManagementResource.java | 59 +-
.../organizations/OrganizationResource.java | 28 +-
.../organizations/OrganizationsResource.java | 23 +-
.../applications/ApplicationResource.java | 61 +-
.../applications/ApplicationsResource.java | 34 +-
.../imports/FileIncludesResource.java | 9 -
.../applications/imports/ImportsResource.java | 83 +-
.../organizations/OrganizationResource.java | 32 +-
.../OAuth2AccessTokenSecurityFilter.java | 2 +-
.../rest/test/RefreshIndexResource.java | 25 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +-
.../apache/usergrid/rest/IndexResourceIT.java | 1 +
.../apache/usergrid/rest/NotificationsIT.java | 93 +-
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../rest/applications/ApplicationCreateIT.java | 135 +
.../rest/applications/ApplicationDeleteIT.java | 389 +++
.../applications/ApplicationDeleteTest.java | 54 -
.../applications/ApplicationResourceIT.java | 100 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/paging/PagingResourceIT.java | 17 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/PermissionsResourceIT.java | 15 -
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 16 +-
.../events/ApplicationRequestCounterIT.java | 2 +-
.../applications/queries/BasicGeoTests.java | 21 +-
.../applications/queries/GeoPagingTest.java | 58 +-
.../applications/queries/MatrixQueryTests.java | 298 +-
.../rest/applications/queries/OrderByTest.java | 2 +-
.../queues/AbstractQueueResourceIT.java | 484 ---
.../queues/QueueResourceLong1IT.java | 149 -
.../queues/QueueResourceLong2IT.java | 99 -
.../queues/QueueResourceLong3IT.java | 201 --
.../queues/QueueResourceShortIT.java | 432 ---
.../rest/filters/ContentTypeResourceIT.java | 55 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +--
.../usergrid/rest/management/AdminUsersIT.java | 64 +-
.../rest/management/ExportResourceIT.java | 674 ++--
.../rest/management/ImportResourceIT.java | 179 +-
.../rest/management/ManagementResourceIT.java | 151 +-
.../rest/management/OrganizationsIT.java | 82 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/README.md | 111 +
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/ApplicationsResource.java | 34 +-
.../endpoints/CollectionEndpoint.java | 13 +
.../endpoints/NamedResource.java | 158 +-
.../endpoints/OrganizationResource.java | 6 +-
.../resource2point0/endpoints/RootResource.java | 1 +
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 54 +-
.../endpoints/mgmt/ApplicationsResource.java | 58 +
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/ManagementResponse.java | 72 +
.../endpoints/mgmt/MeResource.java | 32 +
.../mgmt/OrganizationApplicationResponse.java | 60 +
.../endpoints/mgmt/OrganizationResource.java | 5 +-
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
stack/rest/src/test/resources/log4j.properties | 10 +-
.../resources/testImportCorrect.testCol.1.json | 18 -
...testImportInvalidJson.testApplication.3.json | 153 -
.../testimport-bad-json-testapp.3.json | 153 +
.../src/test/resources/testimport-bad-json.json | 67 +
.../resources/testimport-correct-testcol.1.json | 18 +
.../test/resources/testimport-test-app.2.json | 56 +
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 51 +-
.../usergrid/management/ApplicationInfo.java | 4 +
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ApplicationCreatorImpl.java | 12 +-
.../cassandra/ManagementServiceImpl.java | 236 +-
.../management/export/ExportServiceImpl.java | 4 +-
.../management/importer/FileImportJob.java | 8 +-
.../management/importer/FileImportTracker.java | 6 +-
.../usergrid/management/importer/ImportJob.java | 4 +-
.../management/importer/ImportServiceImpl.java | 88 +-
.../usergrid/security/AuthPrincipalType.java | 3 +-
.../security/providers/AbstractProvider.java | 1 +
.../security/providers/FacebookProvider.java | 6 +-
.../security/providers/FoursquareProvider.java | 6 +-
.../shiro/PrincipalCredentialsToken.java | 20 +-
.../apache/usergrid/security/shiro/Realm.java | 2 +-
.../services/AbstractCollectionService.java | 17 +-
.../services/AbstractConnectionsService.java | 4 +-
.../AbstractPathBasedColllectionService.java | 2 +-
.../usergrid/services/AbstractService.java | 2 +-
.../usergrid/services/ServiceContext.java | 2 +-
.../services/ServiceManagerFactory.java | 4 -
.../usergrid/services/ServiceParameter.java | 3 +-
.../usergrid/services/ServiceRequest.java | 2 +-
.../usergrid/services/ServiceResults.java | 38 +-
.../applications/ApplicationsService.java | 2 +-
.../services/devices/DevicesService.java | 2 +-
.../usergrid/services/groups/GroupsService.java | 2 +-
.../groups/activities/ActivitiesService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../notifications/InactiveDeviceManager.java | 8 +-
.../notifications/NotificationsService.java | 4 +-
.../apns/FailedConnectionListener.java | 31 +-
.../impl/ApplicationQueueManagerImpl.java | 197 +-
.../services/notifications/wns/WNSAdapter.java | 60 +-
.../usergrid/services/roles/RolesService.java | 10 +-
.../usergrid/services/users/UsersService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../org/apache/usergrid/ServiceITSetup.java | 5 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 43 +-
.../migration/AppInfoMigrationPluginTest.java | 231 ++
.../apache/usergrid/management/EmailFlowIT.java | 9 +-
.../usergrid/management/OrganizationIT.java | 16 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ApplicationCreatorIT.java | 14 +-
.../cassandra/ManagementServiceIT.java | 35 +-
.../management/export/ExportServiceIT.java | 20 +-
.../importer/FileImportTrackerTest.java | 12 +-
.../management/importer/ImportCollectionIT.java | 20 +-
.../importer/ImportConnectionsTest.java | 9 +-
.../management/importer/ImportServiceIT.java | 2 +-
.../security/tokens/TokenServiceIT.java | 2 -
.../usergrid/services/ActivitiesServiceIT.java | 2 +-
.../usergrid/services/CollectionServiceIT.java | 2 +
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 4 +-
.../usergrid/services/ServiceFactoryIT.java | 8 +-
.../usergrid/services/ServiceInvocationIT.java | 2 +-
.../usergrid/services/ServiceRequestIT.java | 9 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 4 +-
.../notifications/NotifiersServiceIT.java | 19 +-
.../apns/NotificationsServiceIT.java | 66 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../src/test/resources/log4j.properties | 6 +-
.../resources/usergrid-custom-test.properties | 2 +
.../usergrid/cassandra/SpringResource.java | 13 +-
.../setup/ConcurrentProcessSingleton.java | 49 +-
.../org/apache/usergrid/tools/AppAudit.java | 3 +-
.../org/apache/usergrid/tools/CredExport.java | 2 +-
.../org/apache/usergrid/tools/DupOrgRepair.java | 1 -
.../java/org/apache/usergrid/tools/Import.java | 14 +-
.../usergrid/tools/UniqueIndexCleanup.java | 6 +-
776 files changed, 30517 insertions(+), 39729 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 038be44,23514c8..748a069
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -70,11 -63,9 +63,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
- import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
- import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
- import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
++import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+ import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@@ -88,15 -78,8 +79,9 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
- import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@@ -178,21 -163,20 +165,24 @@@ public class CpEntityManager implement
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
- private UUID applicationId;
+ private final UUID applicationId;
private Application application;
- private ManagerCache managerCache;
- private ApplicationScope applicationScope;
+ private final ManagerCache managerCache;
+
+ private final ApplicationScope applicationScope;
- private CassandraService cass;
+ private final CassandraService cass;
- private CounterUtils counterUtils;
+ private final CounterUtils counterUtils;
+
+ private final AsyncIndexService indexService;
+
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
+
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
private Timer aggCounterTimer;
@@@ -220,14 -207,22 +213,28 @@@
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
- public CpEntityManager() {
-
- }
-
- @Override
- public void init( final CassandraService cassandraService, final CounterUtils counterUtils, final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory, final EntityCollectionManagerFactory entityCollectionManagerFactory, final ManagerCache managerCache, UUID applicationId ) {
+ /**
+ * Fugly, make this part of DI
+ * @param cass
+ * @param counterUtils
+ * @param managerCache
+ * @param metricsFactory
+ * @param applicationId
++ * @param entityCollectionManagerFactory
++ * @param graphManagerFactory
+ */
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
++ public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache,
++ final MetricsFactory metricsFactory,final EntityCollectionManagerFactory entityCollectionManagerFactory,
++ final GraphManagerFactory graphManagerFactory , final UUID applicationId ) {
++ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
++ this.graphManagerFactory = graphManagerFactory;
+ Preconditions.checkNotNull( cass, "cass must not be null" );
+ Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
+ Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
+ Preconditions.checkNotNull( indexService, "indexService must not be null" );
+
this.managerCache = managerCache;
this.applicationId = applicationId;
@@@ -772,9 -733,10 +745,10 @@@
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
- CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, applicationId, entityRef, null, metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory );
- return rmi;
+
+ CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, indexService, this, applicationId, entityRef );
++ new CpRelationManager( metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory, indexService, this, applicationId, entityRef );
+ return relationManager;
}
@@@ -2888,34 -2818,8 +2830,7 @@@
}
-
@Override
- public void refreshIndex() {
-
- // refresh factory indexes
-
-
- // refresh this Entity Manager's application's index
- EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
- ei.refresh();
- }
-
-
- @Override
- public void createIndex() {
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- ei.initializeIndex();
- }
-
- public void deleteIndex(){
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- ei.deleteIndex();
- }
-
-
-
-
-
- @Override
public void flushManagerCaches() {
managerCache.invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5b8e715,f08bce4..6ba71e2
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -177,10 -192,7 +192,8 @@@ public class CpEntityManagerFactory imp
private EntityManager _getEntityManager( UUID applicationId ) {
-
- EntityManager em = new CpEntityManager();
- em.init( cassandraService, counterUtils, metricsFactory, graphManagerFactory, entityCollectionManagerFactory, managerCache, applicationId );
-
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId );
++ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId,
++ entityCollectionManagerFactory, graphManagerFactory );
return em;
}
@@@ -432,17 -436,17 +437,17 @@@
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
- Observable<Edge> edges = gm.loadEdgesFromSource(new SimpleSearchByEdgeType(
- fromEntityId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null));
+ Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
+ fromEntityId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ));
- Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
- while ( iter.hasNext() ) {
+ // TODO This is wrong, and will result in OOM if there are too many applications.
+ // This needs to stream properly with a buffer
- Edge edge = iter.next();
+ edges.doOnNext(edge -> {
Id targetId = edge.getTargetNode();
- logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[] {
+ logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[]{
edge.getSourceNode().getType(), edge.getSourceNode().getUuid(),
edge.getTargetNode().getType(), edge.getTargetNode().getUuid()
});
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index edd352e,ea554a3..8e2fdd7
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -54,86 -50,44 +50,48 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
- import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
- import org.apache.usergrid.persistence.cassandra.IndexUpdate;
- import org.apache.usergrid.persistence.cassandra.QueryProcessorImpl;
- import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
- import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
- import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
- import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
- import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
- import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
- import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
- import org.apache.usergrid.persistence.geo.EntityLocationRef;
- import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+ import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
- import org.apache.usergrid.persistence.index.EntityIndex;
+ import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexScope;
+ import org.apache.usergrid.persistence.index.IndexEdge;
+ import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchTypes;
- import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.Identifier;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.util.UUIDGenerator;
- import org.apache.usergrid.persistence.query.ir.AllNode;
- import org.apache.usergrid.persistence.query.ir.NameIdentifierNode;
- import org.apache.usergrid.persistence.query.ir.QueryNode;
- import org.apache.usergrid.persistence.query.ir.QuerySlice;
- import org.apache.usergrid.persistence.query.ir.SearchVisitor;
- import org.apache.usergrid.persistence.query.ir.WithinNode;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionIndexSliceParser;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
- import org.apache.usergrid.persistence.query.ir.result.EmptyIterator;
- import org.apache.usergrid.persistence.query.ir.result.GeoIterator;
- import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
- import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
import org.apache.usergrid.persistence.schema.CollectionInfo;
- import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
- import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
- import me.prettyprint.hector.api.Keyspace;
- import me.prettyprint.hector.api.beans.DynamicComposite;
- import me.prettyprint.hector.api.beans.HColumn;
- import me.prettyprint.hector.api.mutation.Mutator;
import rx.Observable;
- import rx.functions.Action1;
- import rx.functions.Action2;
import rx.functions.Func1;
- import static java.util.Arrays.asList;
-
- import static me.prettyprint.hector.api.factory.HFactory.createMutator;
- import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchByEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getNameFromEdgeType;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_TYPES;
- import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
- import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE;
@@@ -172,10 -106,7 +110,9 @@@ public class CpRelationManager implemen
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
-
private ManagerCache managerCache;
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
private EntityManager em;
@@@ -194,22 -122,10 +128,11 @@@
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- private Timer createConnectionTimer;
- private Timer cassConnectionDelete;
- private Timer esDeleteConnectionTimer;
- public CpRelationManager() {}
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
++ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final
++ EntityCollectionManagerFactory entityCollectionManagerFactory, final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
- public CpRelationManager init(
- EntityManager em,
- UUID applicationId,
- EntityRef headEntity,
- IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory,
- ManagerCache managerCache,
- EntityCollectionManagerFactory entityCollectionManagerFactory,
- GraphManagerFactory graphManagerFactory ) {
Assert.notNull( em, "Entity manager cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
@@@ -225,30 -141,13 +148,16 @@@
this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
- this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
- this.indexBucketLocator = indexBucketLocator; // TODO: this also
- this.metricsFactory = metricsFactory;
- this.updateCollectionTimer = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
- this.createConnectionTimer = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" );
- this.cassConnectionDelete = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" );
- this.esDeleteConnectionTimer = metricsFactory.getTimer(CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" );
- // load the Core Persistence version of the head entity as well
- this.headEntityScope = getCollectionScopeNameFromEntityType(
- applicationScope.getApplication(), headEntity.getType());
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.graphManagerFactory = graphManagerFactory;
+
+ this.metricsFactory = metricsFactory;
+ this.updateCollectionTimer =
+ metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
+
if ( logger.isDebugEnabled() ) {
- logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[] {
- headEntity.getType(),
- headEntity.getUuid(),
- headEntityScope.getApplication(),
- headEntityScope.getOwner(),
- headEntityScope.getName()
+ logger.debug( "Loading head entity {}:{} from app {}", new Object[] {
+ headEntity.getType(), headEntity.getUuid(), applicationScope
} );
}
@@@ -339,8 -227,7 +237,7 @@@
public Observable<Edge> call( final String edgeType ) {
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
-
}
} );
@@@ -538,24 -295,27 +305,6 @@@
}
- private boolean moreThanOneOutboundConnection( EntityRef source, String connectionType ) {
- private boolean moreThanOneInboundConnection( EntityRef target, String connectionType ) {
--
- Id sourceId = new SimpleId( source.getUuid(), source.getType() );
- Id targetId = new SimpleId( target.getUuid(), target.getType() );
--
-- GraphManager gm = managerCache.getGraphManager( applicationScope );
-
- Observable<Edge> edgesFromSource = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- sourceId,
- CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
- System.currentTimeMillis(),
- SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) ); // last
--
- int count = edgesFromSource.take( 2 ).count().toBlocking().last();
- Observable<Edge> edgesToTarget = gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType( targetId, CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
- System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) ); // last
--
- return count > 1;
- Iterator<Edge> iterator = edgesToTarget.toBlocking().getIterator();
- int count = 0;
- while ( iterator.hasNext() ) {
- iterator.next();
- if ( count++ > 1 ) {
- return true;
- }
- }
- return false;
-- }
--
@Override
public Set<String> getCollections() throws Exception {
@@@ -927,43 -637,29 +626,31 @@@
headEntity = em.validate( headEntity );
- CollectionInfo collection =
- getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
if ( collection == null ) {
- throw new RuntimeException( "Cannot find collection-info for '" + collName
- + "' of " + headEntity.getType() + ":" + headEntity .getUuid() );
+ throw new RuntimeException(
+ "Cannot find collection-info for '" + collName + "' of " + headEntity.getType() + ":" + headEntity
+ .getUuid() );
}
- query.setEntityType( collection.getType() );
- query = adjustQuery( query );
- /**
- * It's a graph query, execute the graph query executor.
- *
- * TODO refactor all of this away into commands
- */
- if(query.isGraphQuery()){
- final QueryExecutor executor = new CollectionGraphQueryExecutor( this.entityCollectionManagerFactory,graphManagerFactory, applicationScope, headEntity, query.getCursor(), collName, query.getLimit() );
- return executor.next();
- }
+ final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
- final IndexScope indexScope = new IndexScopeImpl(
- cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
-
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ logger.debug( "Searching scope {}", searchEdge );
- final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
+ query.setEntityType( collection.getType() );
+ query = adjustQuery( query );
- logger.debug( "Searching scope {}:{}",
- indexScope.getOwner().toString(), indexScope.getName() );
+
+
- final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = new CollectionResultsLoaderFactoryImpl( managerCache );
+ final CollectionResultsLoaderFactoryImpl resultsLoaderFactory =
+ new CollectionResultsLoaderFactoryImpl( managerCache );
//execute the query and return our next result
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index b2354a6,3f2c9d6..15e4341
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@@ -103,56 -103,39 +105,39 @@@ public class CpWalker
edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
}
- Observable<String> edgeTypes = gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( applicationId, edgeType, null ) );
-
- edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
-
- logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
-
- return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- applicationId, edgeType, Long.MAX_VALUE, order , Optional.<Edge>absent() ) );
-
- }
-
- } ).parallel( new Func1<Observable<Edge>, Observable<Edge>>() {
-
- @Override
- public Observable<Edge> call( final Observable<Edge> edgeObservable ) { // process edges in parallel
- return edgeObservable.doOnNext( new Action1<Edge>() { // visit and update then entity
-
- @Override
- public void call( Edge edge ) {
-
- logger.info( "Re-indexing edge {}", edge );
-
- EntityRef targetNodeEntityRef = new SimpleEntityRef(
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid() );
-
- Entity entity;
- try {
- entity = em.get( targetNodeEntityRef );
- }
- catch ( Exception ex ) {
- logger.error( "Error getting sourceEntity {}:{}, continuing",
- targetNodeEntityRef.getType(),
- targetNodeEntityRef.getUuid() );
- return;
- }
- if(entity == null){
- return;
- }
- String collName = CpNamingUtils.getCollectionName( edge.getType() );
- visitor.visitCollectionEntry( em, collName, entity );
- }
- } );
- }
- }, Schedulers.io() )
+ Observable<Edge> edges = gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( applicationId, edgeType, null ) ).flatMap( emittedEdgeType -> {
+
+ logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
+
+ return gm.loadEdgesFromSource(
- new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, null ) );
++ new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, Optional.absent() ) );
+ } ).flatMap( edge -> {
+ //run each edge through it's own scheduler, up to 100 at a time
+ return Observable.just( edge ).doOnNext( edgeValue -> {
+ logger.info( "Re-indexing edge {}", edgeValue );
+
+ EntityRef targetNodeEntityRef =
+ new SimpleEntityRef( edgeValue.getTargetNode().getType(), edgeValue.getTargetNode().getUuid() );
+
+ Entity entity;
+ try {
+ entity = em.get( targetNodeEntityRef );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error getting sourceEntity {}:{}, continuing", targetNodeEntityRef.getType(),
+ targetNodeEntityRef.getUuid() );
+ return;
+ }
+ if ( entity == null ) {
+ return;
+ }
+ String collName = getNameFromEdgeType( edgeValue.getType() );
+ visitor.visitCollectionEntry( em, collName, entity );
+ } ).subscribeOn( Schedulers.io() );
+ }, 100 );
// wait for it to complete
- .toBlocking().lastOrDefault( null ); // end foreach on edges
+ edges.toBlocking().lastOrDefault( null ); // end foreach on edges
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
[22/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' into tempbranch
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into tempbranch
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1c207d76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1c207d76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1c207d76
Branch: refs/heads/USERGRID-593
Commit: 1c207d76e29980b217613231195e58905da5af38
Parents: 5945574 4cc2e0b
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 18:28:01 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 18:28:01 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 +++++--
.../main/resources/usergrid-core-context.xml | 20 +-------------------
.../usergrid/persistence/EntityManagerIT.java | 8 ++------
.../usergrid/persistence/index/IndexFig.java | 15 +++++++++++++++
.../impl/EsApplicationEntityIndexImpl.java | 4 +++-
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
6 files changed, 27 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
[45/52] [abbrv] incubator-usergrid git commit: Removes redundant
logic in metrics factory
Posted by to...@apache.org.
Removes redundant logic in metrics factory
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/34fd9cf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/34fd9cf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/34fd9cf3
Branch: refs/heads/USERGRID-593
Commit: 34fd9cf334b727aac4646c545ebefcb734845306
Parents: 0714b11
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Apr 22 14:45:59 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Apr 22 14:45:59 2015 -0600
----------------------------------------------------------------------
.../core/metrics/MetricsFactoryImpl.java | 44 ++------------------
1 file changed, 4 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/34fd9cf3/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 904e56a..b068782 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -49,7 +49,6 @@ public class MetricsFactoryImpl implements MetricsFactory {
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
- private ConcurrentHashMap<String, Metric> hashMap;
private static final Logger LOG = LoggerFactory.getLogger( MetricsFactoryImpl.class );
@@ -68,7 +67,6 @@ public class MetricsFactoryImpl implements MetricsFactory {
else {
LOG.warn( "MetricsService:Logger not started." );
}
- hashMap = new ConcurrentHashMap<String, Metric>();
jmxReporter = JmxReporter.forRegistry( registry ).build();
jmxReporter.start();
@@ -83,25 +81,25 @@ public class MetricsFactoryImpl implements MetricsFactory {
@Override
public Timer getTimer( Class<?> klass, String name ) {
- return getMetric( Timer.class, klass, name );
+ return getRegistry().timer( MetricRegistry.name( klass, name ) );
}
@Override
public Histogram getHistogram( Class<?> klass, String name ) {
- return getMetric( Histogram.class, klass, name );
+ return getRegistry().histogram( MetricRegistry.name( klass, name ) );
}
@Override
public Counter getCounter( Class<?> klass, String name ) {
- return getMetric( Counter.class, klass, name );
+ return getRegistry().counter( MetricRegistry.name( klass, name ) );
}
@Override
public Meter getMeter( Class<?> klass, String name ) {
- return getMetric( Meter.class, klass, name );
+ return getRegistry().meter( MetricRegistry.name( klass, name ) );
}
@@ -112,38 +110,4 @@ public class MetricsFactoryImpl implements MetricsFactory {
}
- private <T> T getMetric( Class<T> metricClass, Class<?> klass, String name ) {
- String key = metricClass.getName() + klass.getName() + name;
- Metric metric = hashMap.get( key );
- if ( metric == null ) {
- if ( metricClass == Histogram.class ) {
- metric = this.getRegistry().histogram( MetricRegistry.name( klass, name ) );
- }
- if ( metricClass == Timer.class ) {
- metric = this.getRegistry().timer( MetricRegistry.name( klass, name ) );
- }
- if ( metricClass == Meter.class ) {
- metric = this.getRegistry().meter( MetricRegistry.name( klass, name ) );
- }
- if ( metricClass == Counter.class ) {
- metric = this.getRegistry().counter( MetricRegistry.name( klass, name ) );
- }
-
-
- hashMap.put( key, metric );
- }
- return ( T ) metric;
- }
-
-
- /**
- *
- * @param metricClass
- * @param klass
- * @param name
- * @return
- */
- private String getKey( Class<?> metricClass, Class<?> klass, String name ) {
- return metricClass.getName() + klass.getName() + name;
- }
}
[04/52] [abbrv] incubator-usergrid git commit: Updated wiring to
allow spring beans to be used in guice
Posted by to...@apache.org.
Updated wiring to allow spring beans to be used in guice
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6f87fff5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6f87fff5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6f87fff5
Branch: refs/heads/USERGRID-593
Commit: 6f87fff520fa14341fef74c8e85fc8456ddc02a8
Parents: 84d779f
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 11:57:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 11:58:40 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../corepersistence/CpEntityManager.java | 19 +--
.../corepersistence/CpEntityManagerFactory.java | 15 ++-
.../corepersistence/CpRelationManager.java | 12 +-
.../usergrid/corepersistence/GuiceFactory.java | 66 +++++-----
.../usergrid/persistence/PersistenceModule.java | 124 +++----------------
.../main/resources/usergrid-core-context.xml | 8 +-
.../resources/usergrid-custom-test.properties | 2 -
8 files changed, 89 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index ffc7628..c21a1ff 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -82,10 +82,9 @@ cassandra.cluster=Test Cluster
cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
#cassandra.keyspace.strategy=org.apache.cassandra.locator.NetworkTopologyStrategy
-#cassandra.keyspace.strategy.options.replication_factor=1
-#cassandra.keyspace.strategy.options.us-east=1
cassandra.keyspace.replication=replication_factor:1
+#cassandra.keyspace.replication=us-east:3
cassandra.username=
cassandra.password=
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index a615a43..1506551 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -165,17 +166,19 @@ public class CpEntityManager implements EntityManager {
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
- private UUID applicationId;
+ private final UUID applicationId;
private Application application;
- private ManagerCache managerCache;
+ private final ManagerCache managerCache;
- private ApplicationScope applicationScope;
+ private final ApplicationScope applicationScope;
- private CassandraService cass;
+ private final CassandraService cass;
- private CounterUtils counterUtils;
+ private final CounterUtils counterUtils;
+
+ private final IndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -215,16 +218,18 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final IndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
+ Preconditions.checkNotNull( indexService, "indexService must not be null" );
this.managerCache = managerCache;
this.applicationId = applicationId;
+ this.indexService = indexService;
applicationScope = CpNamingUtils.getApplicationScope( applicationId );
@@ -734,7 +739,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, this, applicationId, entityRef );
+ new CpRelationManager( metricsFactory, managerCache, indexService, this, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e03ed47..6ad8616 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -39,6 +39,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
+
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.*;
@@ -109,9 +111,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
+ private final IndexService indexService;
- public CpEntityManagerFactory(
- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) {
+ public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
+ final Injector injector) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@ -120,7 +123,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(getManagementEntityManager());
+ this.indexService = injector.getInstance( IndexService.class );
+ this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
+ getManagementEntityManager() );
+
}
@@ -175,8 +181,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
-
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, managerCache, metricsFactory, applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 3c72b60..a0a44b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
@@ -117,18 +118,23 @@ public class CpRelationManager implements RelationManager {
private org.apache.usergrid.persistence.model.entity.Entity cpHeadEntity;
- private ApplicationScope applicationScope;
+ private final ApplicationScope applicationScope;
+
+ private final IndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager(final MetricsFactory metricsFactory, final ManagerCache managerCache, final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final IndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+
Assert.notNull( em, "Entity manager cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
Assert.notNull( headEntity, "Head entity cannot be null" );
Assert.notNull( headEntity.getUuid(), "Head entity uuid cannot be null" );
+ Assert.notNull( indexService, "indexService cannot be null" );
+
// TODO: this assert should not be failing
//Assert.notNull( indexBucketLocator, "indexBucketLocator cannot be null" );
this.em = em;
@@ -155,6 +161,8 @@ public class CpRelationManager implements RelationManager {
Assert.notNull( cpHeadEntity, String
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
+ this.indexService = indexService;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
index 3a08034..6a87005 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
@@ -22,20 +22,22 @@ package org.apache.usergrid.corepersistence;
import java.util.Properties;
+import javax.inject.Named;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
import org.springframework.beans.factory.FactoryBean;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
+import org.springframework.stereotype.Component;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.persistence.PersistenceModule;
+import com.google.common.base.Preconditions;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.google.inject.Singleton;
import com.netflix.config.ConfigurationManager;
import me.prettyprint.cassandra.service.CassandraHost;
@@ -45,25 +47,28 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator;
/**
* Factory for configuring Guice then returning it
*/
-@Singleton
+@Component
public class GuiceFactory implements FactoryBean<Injector> {
private static final Logger logger = LoggerFactory.getLogger( GuiceFactory.class );
- private final CassandraHostConfigurator chc;
+ @Autowired
+ private CassandraHostConfigurator chc;
- private final Properties systemProperties;
+ @Autowired
+ @Named("properties")
+ private Properties systemProperties;
+ @Autowired
private ApplicationContext applicationContext;
private Injector injector;
-
- public GuiceFactory( final ApplicationContext applicationContext, final CassandraHostConfigurator chc, final Properties systemProperties ) {
- this.chc = chc;
- this.systemProperties = systemProperties;
- this.applicationContext = applicationContext;
+ public GuiceFactory() {
+ // this.chc = chc;
+ // this.systemProperties = systemProperties;
+ // this.applicationContext = applicationContext;
}
@@ -83,15 +88,16 @@ public class GuiceFactory implements FactoryBean<Injector> {
if ( hosts.length == 0 ) {
throw new RuntimeException( "Fatal error: no Cassandra hosts configured" );
}
- String sep = "";
+
for ( CassandraHost host : hosts ) {
if ( StringUtils.isEmpty( host.getHost() ) ) {
throw new RuntimeException( "Fatal error: Cassandra hostname cannot be empty" );
}
- hostsString = hostsString + sep + host.getHost();
- sep = ",";
+ hostsString += host.getHost() + ",";
}
+ hostsString = hostsString.substring( 0, hostsString.length() - 1 );
+
logger.info( "hostsString: " + hostsString );
Properties cpProps = new Properties();
@@ -99,21 +105,14 @@ public class GuiceFactory implements FactoryBean<Injector> {
// Some Usergrid properties must be mapped to Core Persistence properties
cpProps.put( "cassandra.hosts", hostsString );
cpProps.put( "cassandra.port", hosts[0].getPort() );
- cpProps.put( "cassandra.cluster_name", systemProperties.getProperty( "cassandra.cluster" ) );
- String cassRemoteString = ( String ) systemProperties.getProperty( "cassandra.use_remote" );
- if ( cassRemoteString != null && cassRemoteString.equals( "false" ) ) {
- cpProps.put( "cassandra.embedded", "true" );
- }
- else {
- cpProps.put( "cassandra.embedded", "false" );
- }
+ cpProps.put( "cassandra.cluster_name", getAndValidateProperty( "cassandra.cluster" ) );
- cpProps.put( "collections.keyspace.strategy.class",
- systemProperties.getProperty( "cassandra.keyspace.strategy" ) );
+ cpProps
+ .put( "collections.keyspace.strategy.class", getAndValidateProperty( "cassandra.keyspace.strategy" ) );
cpProps.put( "collections.keyspace.strategy.options",
- systemProperties.getProperty( "cassandra.keyspace.replication" ) );
+ getAndValidateProperty( "cassandra.keyspace.replication" ) );
logger.debug( "Set Cassandra properties for Core Persistence: " + cpProps.toString() );
@@ -128,14 +127,23 @@ public class GuiceFactory implements FactoryBean<Injector> {
}
-
- //this is seriously fugly, and needs removed we shouldn't be mixing spring and guice
- injector = Guice.createInjector( new CoreModule( ), new PersistenceModule( applicationContext ) );
+ //we have to inject a couple of spring beans into our Guice. Wire it with PersistenceModule
+ injector = Guice.createInjector( new CoreModule(), new PersistenceModule( applicationContext ) );
return injector;
}
+ private String getAndValidateProperty( final String propName ) {
+
+ final String propValue = systemProperties.getProperty( propName );
+
+ Preconditions.checkNotNull( propValue, propName + " cannot be unset. Set this in your properties" );
+
+ return propValue;
+ }
+
+
@Override
public Class<?> getObjectType() {
return Injector.class;
@@ -146,6 +154,4 @@ public class GuiceFactory implements FactoryBean<Injector> {
public boolean isSingleton() {
return true;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
index 6e58676..70fff90 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
@@ -20,32 +20,15 @@
package org.apache.usergrid.persistence;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.Collection;
-import java.util.Properties;
-
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.context.ApplicationContext;
-import org.apache.usergrid.locking.cassandra.HectorLockManagerImpl;
-import com.google.common.base.Preconditions;
-import com.google.common.io.CharSource;
-import com.google.common.io.Resources;
import com.google.inject.AbstractModule;
-import com.google.inject.Inject;
-import com.google.inject.Provides;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-import com.google.inject.name.Names;
+import com.google.inject.Provider;
import com.google.inject.spring.SpringIntegration;
-import me.prettyprint.cassandra.connection.RoundRobinBalancingPolicy;
-import me.prettyprint.cassandra.service.CassandraHostConfigurator;
-import me.prettyprint.cassandra.service.ThriftCluster;
-import me.prettyprint.hector.api.Cluster;
-
/**
* Replacement for configuration of our spring modules with guice
@@ -66,97 +49,18 @@ public class PersistenceModule extends AbstractModule {
@Override
protected void configure() {
- SpringIntegration.bindAll( binder(), applicationContext );
- }
+ //bind the application context to our guice instance
+
+ final BeanFactory beanFactory = applicationContext.getAutowireCapableBeanFactory();
+ bind( BeanFactory.class).toInstance( beanFactory);
+
+ //create our bridge and put the EMF into guice. Ultimately all spring needs removed and we need to just use guice
+
+ final Provider<EntityManagerFactory> emfProvider = SpringIntegration.fromSpring( EntityManagerFactory.class, "entityManagerFactory" );
+
+ bind( EntityManagerFactory.class ).toProvider( emfProvider );
+ }
-// <bean id="cassandraCluster" class="me.prettyprint.cassandra.service.ThriftCluster">
-// <constructor-arg value="${cassandra.cluster}" />
-// <constructor-arg ref="cassandraHostConfigurator" />
-// </bean>
-// @Provides
-// @Singleton
-// @Inject
-// public Cluster configureThrift( @Named( "cassandra.cluster" ) final String cassCluster,
-// @Named( "cassandra.connections" ) final int cassandraConnections ){
-//
-// final int setSize = cassandraConnections == 0 ? 50: cassandraConnections;
-//
-// CassandraHostConfigurator hostConfigurator = new CassandraHostConfigurator( cassCluster );
-//
-// hostConfigurator.setMaxActive( setSize );
-// hostConfigurator.setLoadBalancingPolicy( new RoundRobinBalancingPolicy() );
-//
-//
-// ThriftCluster thriftCluster = new ThriftCluster(cassCluster, hostConfigurator);
-//
-// return thriftCluster;
-//
-// }
-//
-//
-// @Provides
-// @Singleton
-// @Inject
-// public Properties configureProps(final PropertiesProvider propertiesProvider ){
-//
-// final Properties props = new Properties( );
-//
-// for(final String propFile: propertiesProvider.getPropertiesFiles()){
-//
-// final URL url = Resources.getResource( propFile );
-//
-// Preconditions.checkNotNull( url, "Could not find properties file '" + propFile + "' on the classpath" );
-//
-//
-// final CharSource propsInput = Resources.asCharSource( url, Charset.defaultCharset() );
-// try {
-// props.load( propsInput.openStream() );
-// }
-// catch ( IOException e ) {
-// throw new RuntimeException( "Unable to load properties file '" + propFile + "'", e );
-// }
-// }
-//
-// //bind these properties
-// Names.bindProperties( binder(), props );
-//
-// return props;
-// }
-//
-// @Provides
-// @Singleton
-// @Inject
-// public void configureLocks(final Cluster hectorCluster, @Named("cassandra.lock.keyspace") final String lockKeyspace, @Named("cassandra.lock.keyspace") final String writeCl, final String readCl ){
-//
-//
-// final HectorLockManagerImpl hectorLockManager = new HectorLockManagerImpl();
-//
-//
-////
-//// <bean name="consistencyLevelPolicy" class="me.prettyprint.cassandra.model.ConfigurableConsistencyLevel">
-//// <property name="defaultReadConsistencyLevel" value="${cassandra.readcl}"/>
-//// <property name="defaultWriteConsistencyLevel" value="${cassandra.writecl}"/>
-//// </bean>
-//
-//// <bean name="lockManager" class="org.apache.usergrid.locking.cassandra.HectorLockManagerImpl" >
-//// <property name="cluster" ref="cassandraCluster"/>
-//// <property name="keyspaceName" value="${cassandra.lock.keyspace}"/>
-//// <property name="consistencyLevelPolicy" ref="consistencyLevelPolicy"/>
-//// </bean>
-//
-// }
-//
-//
-// /**
-// * Interface to allow users to provide and inject properties
-// */
-// public interface PropertiesProvider{
-// /**
-// * Get the properties files to load
-// * @return
-// */
-// String[] getPropertiesFiles();
-// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index 0be6d00..f33586f 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -95,10 +95,14 @@
<bean id="injector"
class="org.apache.usergrid.corepersistence.GuiceFactory">
- <constructor-arg ref="cassandraHostConfigurator" />
- <constructor-arg ref="properties" />
</bean>
+ <!-- <bean id="injector"
+ class="org.apache.usergrid.corepersistence.GuiceFactory">
+ <constructor-arg ref="cassandraHostConfigurator" />
+ <constructor-arg ref="properties" />
+ </bean>-->
+
<bean id="cassandraService"
class="org.apache.usergrid.persistence.cassandra.CassandraService" init-method="init" destroy-method="destroy">
<constructor-arg ref="properties" />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6f87fff5/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 93ef28b..703424f 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -14,8 +14,6 @@
# these settings allow tests to run and consistently pass on 16GB MacBook Pro
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
-cassandra.startup=external
-elasticsearch.startup=external
cassandra.timeout=2000
cassandra.connections=1000
[14/52] [abbrv] incubator-usergrid git commit: Added factory for RX
I/O Scheduler.
Posted by to...@apache.org.
Added factory for RX I/O Scheduler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9b9a1b02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9b9a1b02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9b9a1b02
Branch: refs/heads/USERGRID-593
Commit: 9b9a1b02f877ee6f35d7c3a7ba74001a17c237f1
Parents: 48d8060
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 08:01:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 10:36:06 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 5 +-
.../corepersistence/CpEntityManager.java | 9 +-
.../corepersistence/CpRelationManager.java | 57 +-------
.../index/AsyncIndexProvider.java | 15 +--
.../index/AsyncIndexService.java | 5 +-
.../index/InMemoryAsyncIndexService.java | 34 ++---
.../corepersistence/index/IndexServiceImpl.java | 9 +-
.../index/SQSAsyncIndexService.java | 4 +-
.../EntityCollectionManagerFactoryImpl.java | 15 ++-
.../impl/EntityCollectionManagerImpl.java | 38 +++---
.../mvcc/stage/write/RollbackAction.java | 2 -
.../persistence/core/guice/CommonModule.java | 12 ++
.../persistence/core/rx/RxSchedulerFig.java | 60 +++++++++
.../persistence/core/rx/RxTaskScheduler.java | 40 ++++++
.../core/rx/RxTaskSchedulerImpl.java | 129 +++++++++++++++++++
.../index/impl/IndexRefreshCommandImpl.java | 14 +-
.../persistence/index/impl/IndexingUtils.java | 6 +-
.../persistence/index/usergrid-mappings.json | 4 +-
18 files changed, 314 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 4758456..6ebff53 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -34,6 +34,9 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
+import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -144,7 +147,7 @@ public class CoreModule extends AbstractModule {
*****/
- bind(IndexService.class).to(IndexServiceImpl.class);
+ bind(IndexService.class).to( IndexServiceImpl.class );
//bind the queue provider
bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 844892a..6c3989d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -620,9 +620,8 @@ public class CpEntityManager implements EntityManager {
}
// update in all containing collections and connection indexes
- CpRelationManager rm = ( CpRelationManager ) getRelationManager( entity );
- rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
- timer.stop();
+
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity );
}
@@ -1067,9 +1066,7 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
- // update in all containing collections and connection indexes
- CpRelationManager rm = ( CpRelationManager ) getRelationManager( entityRef );
- rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index db9816e..be8605e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -255,47 +255,6 @@ public class CpRelationManager implements RelationManager {
}
- public void updateContainingCollectionAndCollectionIndexes(
- final org.apache.usergrid.persistence.model.entity.Entity cpEntity ) {
-
-
- throw new UnsupportedOperationException( "Use the new interface" );
-
-// final GraphManager gm = managerCache.getGraphManager( applicationScope );
-//
-// // loop through all types of edge to target
-//
-//
-// final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
-//
-// final EntityIndexBatch entityIndexBatch = ei.createBatch();
-//
-// final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
-//
-// // for each edge type, emit all the edges of that type
-// .flatMap( etype -> gm.loadEdgesToTarget(
-// new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
-// SearchByEdgeType.Order.DESCENDING, null ) ) )
-//
-// //for each edge we receive index and add to the batch
-// .doOnNext( edge -> {
-// // reindex the entity in the source entity's collection or connection index
-//
-// IndexEdge indexScope = generateScopeFromSource( edge );
-//
-// entityIndexBatch.index( indexScope, cpEntity );
-//
-// } ).doOnCompleted( () -> {
-// Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
-// entityIndexBatch.execute();
-// timeElasticIndexBatch.stop();
-// } ).count().toBlocking().lastOrDefault( 0 );
-//
-// //Adding graphite metrics
-//
-//
-// logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
- }
@Override
@@ -473,8 +432,6 @@ public class CpRelationManager implements RelationManager {
gm.writeEdge( edge ).toBlocking().last();
- //This is broken and needs fixed updateContainingCollectionAndCollectionIndexes See USERGRID-541
-
//perform indexing
@@ -482,7 +439,7 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- indexService.queueEntityIndexUpdate( applicationScope, memberEntity.getId(), memberEntity.getVersion() );
+ indexService.queueEntityIndexUpdate( applicationScope, memberEntity);
if ( logger.isDebugEnabled() ) {
@@ -490,17 +447,7 @@ public class CpRelationManager implements RelationManager {
itemRef.getUuid().toString(), itemRef.getType(), collName
} );
}
- // logger.debug("With head entity scope is {}:{}:{}", new Object[] {
- // headEntityScope.getApplication().toString(),
- // headEntityScope.getOwner().toString(),
- // headEntityScope.getName()});
-
- if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
- throw new UnsupportedOperationException( "Implement me directly in graph " );
-// getRelationManager( itemEntity )
-// .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
-// getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
- }
+
return itemEntity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 8257c94..77b4990 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.index;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
@@ -37,24 +37,23 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final QueryFig queryFig;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private final QueueManagerFactory queueManagerFactory;
private final MetricsFactory metricsFactory;
private final IndexService indexService;
+ private final RxTaskScheduler rxTaskScheduler;
private AsyncIndexService asyncIndexService;
@Inject
- public AsyncIndexProvider( final QueryFig queryFig,
- final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final QueueManagerFactory queueManagerFactory, final MetricsFactory metricsFactory,
- final IndexService indexService ) {
+ public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory, final
+ MetricsFactory metricsFactory,
+ final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
this.queryFig = queryFig;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -77,7 +76,7 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, entityCollectionManagerFactory );
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler );
case SQS:
return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
index d1f2fb6..06310ae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -40,8 +40,7 @@ public interface AsyncIndexService {
* We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
* After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
* @param applicationScope
- * @param entityId
- * @param version
+ * @param entity The entity to index
*/
- void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
index fdc9c65..3e2a271 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -20,56 +20,40 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import rx.Observable;
-import rx.schedulers.Schedulers;
@Singleton
public class InMemoryAsyncIndexService implements AsyncIndexService {
private final IndexService indexService;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final RxTaskScheduler rxTaskScheduler;
@Inject
- public InMemoryAsyncIndexService( final IndexService indexService,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {this.indexService = indexService;
+ public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ this.indexService = indexService;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
- final UUID version ) {
-
- final IndexEntityEvent event = new IndexEntityEvent( applicationScope, entityId, version );
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
//process the entity immediately
//only process the same version, otherwise ignore
- getEntity( applicationScope, entityId).filter( entity-> version.equals(entity.hasVersion() )).doOnNext( entity -> {
- indexService.indexEntity( applicationScope, entity );
- } ).subscribeOn( Schedulers.io() ).subscribe();
-
-
- }
-
- private Observable<Entity> getEntity( final ApplicationScope applicationScope, final Id entityId){
-
- final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- return ecm.load( entityId );
+ Observable.just( toIndex ).doOnNext( entity -> {
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 2a7533a..873e2b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -83,8 +83,8 @@ public class IndexServiceImpl implements IndexService {
//we always index in the target scope
final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
- //we may have to index
- final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeToTarget( edge ) );
+ //we may have to index we're indexing from source->target here
+ final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) );
//we might or might not need to index from target-> source
@@ -141,8 +141,9 @@ public class IndexServiceImpl implements IndexService {
/**
* An observable of sizes as we execute batches
+ *
+ * we're indexing from target->source here
*/
- return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
- .map( edge -> generateScopeFromSource( edge ) );
+ return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection ).map( edge -> generateScopeToTarget( edge ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
index cad20bd..dfcb97a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.QueueManager;
@@ -256,8 +257,7 @@ public class SQSAsyncIndexService implements AsyncIndexService {
@Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
- final UUID version ) {
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 9191c06..761c4b5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerial
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -74,6 +75,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final TaskExecutor taskExecutor;
private final EntityCacheFig entityCacheFig;
private final MetricsFactory metricsFactory;
+ private final RxTaskScheduler rxTaskScheduler;
private LoadingCache<ApplicationScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
@@ -84,7 +86,8 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
writeStart, writeVerifyUnique,
writeOptimisticVerify, writeCommit, rollback, markStart, markCommit,
entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory );
+ mvccLogEntrySerializationStrategy, keyspace,entityVersionTaskFactory, taskExecutor, scope, metricsFactory,
+ rxTaskScheduler );
final EntityCollectionManager proxy = new CachedEntityCollectionManager(entityCacheFig, target );
@@ -95,8 +98,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
@Inject
- public EntityCollectionManagerFactoryImpl( final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
+ public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
final WriteOptimisticVerify writeOptimisticVerify,
final WriteCommit writeCommit, final RollbackAction rollback,
final MarkStart markStart, final MarkCommit markCommit,
@@ -105,9 +107,9 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace,
final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- final EntityCacheFig entityCacheFig,
- MetricsFactory metricsFactory) {
+ @CollectionTaskExecutor final TaskExecutor taskExecutor, final
+ EntityCacheFig entityCacheFig,
+ MetricsFactory metricsFactory, final RxTaskScheduler rxTaskScheduler ) {
this.writeStart = writeStart;
this.writeVerifyUnique = writeVerifyUnique;
@@ -124,6 +126,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.taskExecutor = taskExecutor;
this.entityCacheFig = entityCacheFig;
this.metricsFactory = metricsFactory;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@Override
public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index f0b070c..6f10e86 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -49,6 +49,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSeria
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
@@ -109,6 +110,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final EntityVersionTaskFactory entityVersionTaskFactory;
private final TaskExecutor taskExecutor;
+ private final RxTaskScheduler rxTaskScheduler;
+
private final Keyspace keyspace;
private final Timer writeTimer;
private final Meter writeMeter;
@@ -125,26 +128,21 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Inject
- public EntityCollectionManagerImpl(
- final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit,
- final RollbackAction rollback,
- final MarkStart markStart,
- final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final EntityVersionTaskFactory entityVersionTaskFactory,
- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final ApplicationScope applicationScope,
- final MetricsFactory metricsFactory
-
- ) {
+ public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify, final WriteCommit
+ writeCommit,
+ final RollbackAction rollback, final MarkStart markStart,
+ final MarkCommit markCommit, final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final Keyspace keyspace, final EntityVersionTaskFactory entityVersionTaskFactory,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor, @Assisted final ApplicationScope applicationScope,
+ final MetricsFactory metricsFactory,
+
+ final RxTaskScheduler rxTaskScheduler ) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
+ this.rxTaskScheduler = rxTaskScheduler;
ValidationUtils.validateApplicationScope( applicationScope );
@@ -453,13 +451,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( writeVerifyUnique );
// optimistic verification
Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ Observable.just( mvccEntityCollectionIoEvent ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( writeOptimisticVerify );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
index cd15c26..8342e55 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
@@ -49,7 +49,6 @@ public class RollbackAction implements Action1<Throwable> {
private static final Logger log = LoggerFactory.getLogger( RollbackAction.class );
- private final Scheduler scheduler;
private final UniqueValueSerializationStrategy uniqueValueStrat;
private final MvccLogEntrySerializationStrategy logEntryStrat;
@@ -58,7 +57,6 @@ public class RollbackAction implements Action1<Throwable> {
public RollbackAction(MvccLogEntrySerializationStrategy logEntryStrat,
UniqueValueSerializationStrategy uniqueValueStrat ) {
- scheduler = Schedulers.io();
this.uniqueValueStrat = uniqueValueStrat;
this.logEntryStrat = logEntryStrat;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
index bc84b6b..f266643 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/CommonModule.java
@@ -36,6 +36,9 @@ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerFig;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerImpl;
+import org.apache.usergrid.persistence.core.rx.RxSchedulerFig;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.rx.RxTaskSchedulerImpl;
import com.google.inject.AbstractModule;
import com.google.inject.Key;
@@ -86,6 +89,15 @@ public class CommonModule extends AbstractModule {
//do multibindings for migrations
//create the empty multibinder so other plugins can use it
Multibinder.newSetBinder( binder(), MigrationPlugin.class);
+
+
+ /**
+ * RX java scheduler configuration
+ */
+
+ install ( new GuicyFigModule( RxSchedulerFig.class ));
+
+ bind( RxTaskScheduler.class).to( RxTaskSchedulerImpl.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
new file mode 100644
index 0000000..7986132
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ *
+ */
+public interface RxSchedulerFig extends GuicyFig {
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String IO_SCHEDULER_THREADS = "scheduler.io.threads";
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String IO_SCHEDULER_NAME = "scheduler.io.poolName";
+
+
+
+
+ @Default( "100" )
+ @Key( IO_SCHEDULER_THREADS )
+ int getMaxIoThreads();
+
+ @Default( "Usergrid-RxIOPool" )
+ @Key(IO_SCHEDULER_NAME)
+ String getIoSchedulerName();
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
new file mode 100644
index 0000000..d6cc5e8
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskScheduler.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import org.apache.usergrid.persistence.core.task.Task;
+
+import rx.Scheduler;
+
+
+/**
+ * An interface for returning task schedulers
+ */
+public interface RxTaskScheduler {
+
+ /**
+ * Get the scheduler for tasks that perform blocking I/O
+ * @return
+ */
+ Scheduler getAsyncIOScheduler();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
new file mode 100644
index 0000000..219cde6
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.rx;
+
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Scheduler;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * An implementation of the task scheduler that allows us to control the number of I/O threads
+ */
+@Singleton
+public class RxTaskSchedulerImpl implements RxTaskScheduler {
+
+ private static final Logger log = LoggerFactory.getLogger( RxTaskSchedulerImpl.class );
+
+ private final Scheduler scheduler;
+ private final String poolName;
+
+ @Inject
+ public RxTaskSchedulerImpl(final RxSchedulerFig schedulerFig){
+
+ this.poolName = schedulerFig.getIoSchedulerName();
+
+ final int poolSize = schedulerFig.getMaxIoThreads();
+
+
+ final BlockingQueue<Runnable> queue = new ArrayBlockingQueue<Runnable>(poolSize);
+
+
+ final MaxSizeThreadPool threadPool = new MaxSizeThreadPool( queue, poolSize );
+
+
+ this.scheduler = Schedulers.from(threadPool);
+
+
+ }
+
+
+ @Override
+ public Scheduler getAsyncIOScheduler() {
+ return scheduler;
+ }
+
+
+ /**
+ * Create a thread pool that will reject work if our audit tasks become overwhelmed
+ */
+ private final class MaxSizeThreadPool extends ThreadPoolExecutor {
+
+ public MaxSizeThreadPool( final BlockingQueue<Runnable> queue, final int maxPoolSize ) {
+
+ super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ),
+ new RejectedHandler() );
+ }
+ }
+
+
+ /**
+ * Thread factory that will name and count threads for easier debugging
+ */
+ private final class CountingThreadFactory implements ThreadFactory {
+
+ private final AtomicLong threadCounter = new AtomicLong();
+
+
+ @Override
+ public Thread newThread( final Runnable r ) {
+ final long newValue = threadCounter.incrementAndGet();
+
+ Thread t = new Thread( r, poolName + "-" + newValue );
+
+ t.setDaemon( true );
+
+ return t;
+ }
+ }
+
+
+ /**
+ * The handler that will handle rejected executions and signal the interface
+ */
+ private final class RejectedHandler implements RejectedExecutionHandler {
+
+
+ @Override
+ public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
+ log.warn( "{} task queue full, rejecting task {}", poolName, r );
+
+ //TODO T.N. do we want to run this on the caller thread?
+
+ throw new RejectedExecutionException( "Unable to run task, queue full" );
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index 07150bb..c997e5a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
@@ -48,7 +49,6 @@ import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import rx.Observable;
-import rx.schedulers.Schedulers;
import rx.util.async.Async;
@@ -64,12 +64,13 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer timer;
-
+ private final RxTaskScheduler rxTaskScheduler;
@Inject
public IndexRefreshCommandImpl( IndexIdentifier indexIdentifier, EsProvider esProvider,
IndexBufferConsumer producer, IndexFig indexFig, MetricsFactory metricsFactory,
- final IndexCache indexCache ) {
+ final IndexCache indexCache, final RxTaskScheduler rxTaskScheduler ) {
+
this.timer = metricsFactory.getTimer( IndexRefreshCommandImpl.class, "index.refresh.timer" );
@@ -78,6 +79,7 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
this.producer = producer;
this.indexFig = indexFig;
this.indexCache = indexCache;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -141,14 +143,14 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
logger.error( "Failed during refresh search for " + uuid, ee );
throw new RuntimeException( "Failed during refresh search for " + uuid, ee );
}
- }, Schedulers.io() ).call();
+ }, rxTaskScheduler.getAsyncIOScheduler() ).call();
return future.doOnNext( found -> {
if ( !found.hasFinished() ) {
- logger.error(String.format("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ logger.error("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime());
}else{
- logger.info(String.format("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ logger.info("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime());
}
} ).doOnCompleted(() -> {
//clean up our data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 7c33084..de5e29d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -54,11 +54,11 @@ public class IndexingUtils {
public static final String ENTITY_TYPE_FIELDNAME = "entityType";
- public static final String EDGE_NODE_ID_FIELDNAME = "edgeNodeId";
+ public static final String EDGE_NODE_ID_FIELDNAME = "nodeId";
public static final String EDGE_NAME_FIELDNAME = "edgeName";
- public static final String EDGE_NODE_TYPE_FIELDNAME = "edgeType";
+ public static final String EDGE_NODE_TYPE_FIELDNAME = "entityNodeType";
public static final String EDGE_TIMESTAMP_FIELDNAME = "edgeTimestamp";
@@ -107,8 +107,6 @@ public class IndexingUtils {
idString( sb, scope.getNodeId() );
sb.append( FIELD_SEPERATOR );
sb.append( scope.getEdgeName() );
- sb.append( FIELD_SEPERATOR );
- sb.append( scope.getNodeType() );
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9b9a1b02/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index 4da1902..d4c34cd 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -21,7 +21,7 @@
"index": "not_analyzed",
"doc_values": true
},
- "edgeNodeId": {
+ "nodeId": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
@@ -31,7 +31,7 @@
"index": "not_analyzed",
"doc_values": true
},
- "edgeType": {
+ "entityNodeType": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
[39/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into USERGRID-578
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-578
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1fc18b56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1fc18b56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1fc18b56
Branch: refs/heads/USERGRID-593
Commit: 1fc18b56fe94484c56c4b6c4bb79f47a2f8e4b59
Parents: 46a0e8f 2a58270
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 14:59:42 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 14:59:42 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/mq/cassandra/QueuesCF.java | 5 -
.../persistence/cassandra/ApplicationCF.java | 37 ------
.../cassandra/CassandraPersistenceUtils.java | 122 +------------------
3 files changed, 1 insertion(+), 163 deletions(-)
----------------------------------------------------------------------
[42/52] [abbrv] incubator-usergrid git commit: Updated tests and
providers for testing all async index services and index service itself
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index b0f3995..8be93f6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -20,40 +20,54 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.List;
import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.EsRunner;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.index.impl.IndexRequest;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
import rx.Observable;
+import rx.schedulers.Schedulers;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
public class IndexServiceTest {
@Inject
@@ -63,24 +77,34 @@ public class IndexServiceTest {
@Inject
public GraphManagerFactory graphManagerFactory;
+ @Inject
+ public EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+ @Inject
+ public EntityIndexFactory entityIndexFactory;
+
+ @Inject
+ public IndexFig indexFig;
+
public GraphManager graphManager;
public ApplicationScope applicationScope;
+
@Before
- public void setup(){
- applicationScope = getApplicationScope( UUIDGenerator.newTimeUUID());
+ public void setup() {
+ applicationScope = getApplicationScope( UUIDGenerator.newTimeUUID() );
graphManager = graphManagerFactory.createEdgeManager( applicationScope );
}
@Test
- public void testSingleIndexFromSource(){
- final Entity entity = new Entity( createId( "test" ), UUIDGenerator.newTimeUUID());
+ public void testSingleIndexFromSource() {
+ final Entity entity = new Entity( createId( "test" ), UUIDGenerator.newTimeUUID() );
entity.setField( new StringField( "string", "foo" ) );
- final Edge collectionEdge = createCollectionEdge( applicationScope.getApplication(), "tests", entity.getId() );
+ final Edge collectionEdge = createCollectionEdge( applicationScope.getApplication(), "tests", entity.getId() );
//write the edge
graphManager.writeEdge( collectionEdge ).toBlocking().last();
@@ -91,22 +115,199 @@ public class IndexServiceTest {
//real users should never call to blocking, we're not sure what we'll get
- final IndexOperationMessage results = indexed.toBlocking().last();
+ final IndexOperationMessage results = indexed.toBlocking().last();
final Set<IndexRequest> indexRequests = results.getIndexRequests();
//ensure our value made it to the index request
final IndexRequest indexRequest = indexRequests.iterator().next();
- assertNotNull(indexRequest);
+ assertNotNull( indexRequest );
+ }
+
+
+
+// @Test( timeout = 60000 )
+ @Test( )
+ public void testSingleCollectionConnection() throws InterruptedException {
+
+
+ ApplicationScope applicationScope =
+ new ApplicationScopeImpl( new SimpleId( UUID.randomUUID(), "application" ) );
+
+
+ final Entity testEntity = new Entity( createId( "thing" ), UUIDGenerator.newTimeUUID() );
+ testEntity.setField( new StringField( "string", "foo" ) );
+
+
+ //write the entity before indexing
+ final EntityCollectionManager collectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ collectionManager.write( testEntity ).toBlocking().last();
+
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //create our collection edge
+ final Edge collectionEdge =
+ CpNamingUtils.createCollectionEdge( applicationScope.getApplication(), "things", testEntity.getId() );
+ graphManager.writeEdge( collectionEdge ).toBlocking().last();
+
+
+
+ final Id connectingId = createId( "connecting" );
+ final Edge edge = CpNamingUtils.createConnectionEdge( connectingId, "likes", testEntity.getId() );
+
+
+ final Edge connectionSearch = graphManager.writeEdge( edge ).toBlocking().last();
+
+
+
+
+ //now index
+ final int batches = indexService.indexEntity( applicationScope, testEntity ).count().toBlocking().last();
+
+
+ assertEquals(1, batches);
+
+ final ApplicationEntityIndex applicationEntityIndex =
+ entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+ final SearchEdge collectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( collectionEdge );
+
+ final CountDownLatch latch = new CountDownLatch( 1 );
+
+
+ //query until it's available
+ final CandidateResults collectionResults = getResults( applicationEntityIndex, collectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, collectionResults.size() );
+
+ assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
+
+ latch.await();
+
+ final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
+
+ //query until it's available
+ final CandidateResults connectionResults = getResults( applicationEntityIndex, connectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+ assertEquals( 1, connectionResults.size() );
-// assertEquals(applicationScope.getApplication(), indexRequest.);
-// assertEquals(collectionEdge.getTimestamp(), edge.getTimestamp());
-// assertEquals(collectionEdge.getType(), edge.getEdgeName());
-// assertEquals( SearchEdge.NodeType.TARGET, edge.getNodeType());
+ assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
}
+ /**
+ * Tests that when we have large connections, we batch appropriately
+ * @throws InterruptedException
+ */
+// @Test( timeout = 60000 )
+ @Test( )
+ public void testConnectingIndexingBatches() throws InterruptedException {
+
+ ApplicationScope applicationScope =
+ new ApplicationScopeImpl( new SimpleId( UUID.randomUUID(), "application" ) );
+
+
+ final Entity testEntity = new Entity( createId( "thing" ), UUIDGenerator.newTimeUUID() );
+ testEntity.setField( new StringField( "string", "foo" ) );
+
+
+ //write the entity before indexing
+ final EntityCollectionManager collectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ collectionManager.write( testEntity ).toBlocking().last();
+
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //create our collection edge
+ final Edge collectionEdge =
+ CpNamingUtils.createCollectionEdge( applicationScope.getApplication(), "things", testEntity.getId() );
+ graphManager.writeEdge( collectionEdge ).toBlocking().last();
+
+
+ /**
+ * Write 10k edges 10 at a time in parallel
+ */
+
+ final int edgeCount = 2000;
+
+ final Edge connectionSearch = Observable.range( 0, edgeCount ).flatMap( integer -> {
+ final Id connectingId = createId( "connecting" );
+ final Edge edge = CpNamingUtils.createConnectionEdge( connectingId, "likes", testEntity.getId() );
+
+ return graphManager.writeEdge( edge ).subscribeOn( Schedulers.io() );
+ }, 10 ).toBlocking().last();
+
+
+ //now index
+ final int batches = indexService.indexEntity( applicationScope, testEntity ).count().toBlocking().last();
+
+ //take our edge count + 1 and divided by batch sizes
+ final int expectedSize = ( int ) Math.ceil( ( (double)edgeCount + 1 ) / indexFig.getIndexBatchSize() );
+
+ assertEquals(expectedSize, batches);
+
+ final ApplicationEntityIndex applicationEntityIndex =
+ entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+ final SearchEdge collectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( collectionEdge );
+
+ final CountDownLatch latch = new CountDownLatch( 1 );
+
+
+ //query until it's available
+ final CandidateResults collectionResults = getResults( applicationEntityIndex, collectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, collectionResults.size() );
+
+ assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
+
+ latch.await();
+
+ final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
+
+
+ //query until it's available
+ final CandidateResults connectionResults = getResults( applicationEntityIndex, connectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, connectionResults.size() );
+
+ assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
+ }
+
+
+ private CandidateResults getResults( final ApplicationEntityIndex applicationEntityIndex,
+ final SearchEdge searchEdge, final SearchTypes searchTypes, final String ql,
+ final int count, final int expectedSize, final int attempts ) {
+
+
+ for ( int i = 0; i < attempts; i++ ) {
+ final CandidateResults candidateResults =
+ applicationEntityIndex.search( searchEdge, searchTypes, "select *", 100 );
+
+ if ( candidateResults.size() == expectedSize ) {
+ return candidateResults;
+ }
+
+ try {
+ Thread.sleep( 100 );
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+ }
+
+ fail( "Could not find candidates of size " + expectedSize + "after " + attempts + " attempts" );
+
+ //we'll never reach this, required for compile
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
index 5e9a5a1..aa95ae8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
@@ -20,146 +20,87 @@
package org.apache.usergrid.corepersistence.index;
-import org.apache.usergrid.corepersistence.TestIndexModule;
-import org.apache.usergrid.persistence.index.impl.EsRunner;
+import java.util.UUID;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import com.google.inject.Inject;
import net.jcip.annotations.NotThreadSafe;
-import static org.junit.Assert.assertTrue;
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import static org.junit.Assume.assumeTrue;
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
@NotThreadSafe
-public class SQSAsyncIndexServiceTest {
-
+public class SQSAsyncIndexServiceTest extends AsyncIndexServiceTest {
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
@Rule
public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+
@Inject
public QueueManagerFactory queueManagerFactory;
@Inject
- public QueryFig queryFig;
+ public IndexProcessorFig indexProcessorFig;
@Inject
public MetricsFactory metricsFactory;
+ @Inject
+ public IndexService indexService;
- private SQSAsyncReIndexService bufferQueueSQS;
-
- @Before
- public void setup(){
- bufferQueueSQS = new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
- }
-
-
-
+ @Inject
+ public RxTaskScheduler rxTaskScheduler;
- @Test
- public void testMessageIndexing(){
-
- fail("fix me");
-// ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
-// final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
-// assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
-// assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
-//
-// final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
-// final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
-//
-//
-// final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
-// final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
-//
-//
-// //de-index request
-// final DeIndexRequest
-// deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
-//
-//
-// SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
-//
-// final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
-// SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
-//
-//
-//
-//
-// IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
-// indexOperationMessage.addIndexRequest( indexRequest1);
-// indexOperationMessage.addIndexRequest( indexRequest2);
-//
-// indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
-// indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
-//
-// bufferQueueSQS.offer( indexOperationMessage );
-//
-// //wait for it to send to SQS
-// indexOperationMessage.getFuture().get();
-//
-// //now get it back
-//
-// final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
-//
-// assertTrue(ops.size() > 0);
-//
-// final IndexOperationMessage returnedOperation = ops.get( 0 );
-//
-// //get the operations out
-//
-// final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
-//
-// assertTrue(indexRequestSet.contains(indexRequest1));
-// assertTrue(indexRequestSet.contains(indexRequest2));
-//
-//
-// final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
-//
-// assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
-// assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
-//
-//
-//
-// //now ack the message
-//
-// bufferQueueSQS.ack( ops );
+ @Override
+ protected AsyncIndexService getAsyncIndexService() {
+ return new SQSAsyncIndexService( queueManagerFactory, indexProcessorFig, metricsFactory, indexService,
+ entityCollectionManagerFactory, rxTaskScheduler );
}
-// private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
-// final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-//
-// List<IndexOperationMessage> ops;
-//
-// do{
-// ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
-// }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
-//
-// return ops;
-// }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
index b83b806..ccb235b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/migration/EntityIdScope.java
@@ -20,17 +20,31 @@
package org.apache.usergrid.persistence.collection.serialization.impl.migration;
+import java.io.Serializable;
+
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+
/**
* Tuple containing collectionscope and entityid
*/
-public class EntityIdScope{
- private final Id id;
- private final ApplicationScope applicationScope;
+public class EntityIdScope implements Serializable {
+ private Id id;
+ private ApplicationScope applicationScope;
+
- public EntityIdScope(ApplicationScope applicationScope, Id id){
+ /**
+ * DO NOT DELETE!! Required for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public EntityIdScope() {
+ }
+
+
+ public EntityIdScope( ApplicationScope applicationScope, Id id ) {
this.id = id;
this.applicationScope = applicationScope;
}
@@ -44,4 +58,22 @@ public class EntityIdScope{
public ApplicationScope getApplicationScope() {
return applicationScope;
}
+
+
+ /**
+ * DO NOT DELETE!! Required for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public void setApplicationScope( final ApplicationScope applicationScope ) {
+ this.applicationScope = applicationScope;
+ }
+
+
+ /**
+ * DO NOT DELETE!! Required for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public void setId( final Id id ) {
+ this.id = id;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
index 806768e..47ddf4d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationRelationship.java
@@ -59,13 +59,19 @@ public class MigrationRelationship<T extends VersionedData> {
/**
- * Return true if this is the migration relationship we should use. The version matches the from
- * and is <= the to
- * @param currentVersion
- * @return
+ * Return true if this is the migration relationship we should use. The version matches the from and is <= the to
+ *
+ * @return The span. Minimum span should be used. Integer.MAX_VALUE means this span is unsupported.
*/
- public boolean correctRelationship(final int currentVersion){
- return currentVersion == fromVersion && currentVersion <= toVersion;
+ public int getSpan( final int currentVersion ) {
+
+ //current version is in our range, find it's delta from our min version
+ if ( currentVersion >= fromVersion && currentVersion <= toVersion ) {
+ //we return the fromVersion we're closest to. Distance from 0 is what matters, so
+ return Math.abs( fromVersion - currentVersion );
+ }
+
+ return Integer.MAX_VALUE;
}
@@ -97,4 +103,15 @@ public class MigrationRelationship<T extends VersionedData> {
result = 31 * result + to.hashCode();
return result;
}
+
+
+ @Override
+ public String toString() {
+ return "MigrationRelationship{" +
+ "from=" + from +
+ ", to=" + to +
+ ", fromVersion=" + fromVersion +
+ ", toVersion=" + toVersion +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
index cadd919..dd0e979 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSet.java
@@ -83,35 +83,46 @@ public class VersionedMigrationSet<T extends VersionedData> {
/**
* Get the migration relationship based on our current version. This will return a range that includes the current
* system version as the source, and the highest version we can roll to in the to field
- * @param currentVersion
+ *
* @return The MigrationRelationship. Note the from and the to could be the same version in a current system.
*/
- public MigrationRelationship<T> getMigrationRelationship( final int currentVersion ){
+ public MigrationRelationship<T> getMigrationRelationship( final int currentVersion ) {
final MigrationRelationship<T> relationship = cacheVersion.get( currentVersion );
- if(relationship != null){
+ if ( relationship != null ) {
return relationship;
}
- //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not a big deal
+ //not there, find it. Not the most efficient, but it happens once per version, which rarely changes, so not
+ // a big deal
+ int lastSpan = Integer.MAX_VALUE;
+ MigrationRelationship<T> toUse = null;
- for(MigrationRelationship<T> current: orderedVersions){
+ for ( MigrationRelationship<T> current : orderedVersions ) {
//not our instance, the from is too high
//our from is this instance, so we support this tuple. Our future is >= as well, so we can perform this I/O
- if ( current.correctRelationship( currentVersion )) {
- cacheVersion.put( currentVersion, current );
- return current;
- }
+ final int newSpan = current.getSpan( currentVersion );
+
+ if ( newSpan < lastSpan ) {
+ lastSpan = newSpan;
+ toUse = current;
+ }
}
//if we get here, something is wrong
- throw new IllegalArgumentException( "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions.get( orderedVersions.size()-1 ) );
+ if ( lastSpan == Integer.MAX_VALUE ) {
+ throw new IllegalArgumentException(
+ "Could not find a migration version for version " + currentVersion + " min found was " + orderedVersions
+ .get( orderedVersions.size() - 1 ) );
+ }
+ cacheVersion.put( currentVersion, toUse );
+ return toUse;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
index 7986132..ac45967 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxSchedulerFig.java
@@ -35,14 +35,14 @@ public interface RxSchedulerFig extends GuicyFig {
* Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
* backpressure
*/
- public static final String IO_SCHEDULER_THREADS = "scheduler.io.threads";
+ String IO_SCHEDULER_THREADS = "scheduler.io.threads";
/**
* Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
* backpressure
*/
- public static final String IO_SCHEDULER_NAME = "scheduler.io.poolName";
+ String IO_SCHEDULER_NAME = "scheduler.io.poolName";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
index 5d22eff..aa879c9 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
@@ -22,11 +22,15 @@ package org.apache.usergrid.persistence.core.scope;
import java.io.Serializable;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
/**
* A scope used for organizations
*/
+@JsonDeserialize(as = ApplicationScopeImpl.class)
public interface ApplicationScope extends Serializable {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
index 5733e3a..45a50ba 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScopeImpl.java
@@ -31,9 +31,18 @@ import com.google.common.base.Preconditions;
*/
public class ApplicationScopeImpl implements ApplicationScope {
- protected final Id application;
+ protected Id application;
+
+ /**
+ * Do not delete! Needed for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public ApplicationScopeImpl(){
+
+ }
+
public ApplicationScopeImpl( final Id application ) {
Preconditions.checkNotNull(application, "application id is required");
this.application = application;
@@ -46,6 +55,16 @@ public class ApplicationScopeImpl implements ApplicationScope {
}
+
+ /**
+ * Do not delete! Needed for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public void setApplication( final Id application ) {
+ this.application = application;
+ }
+
+
@Override
public boolean equals( final Object o ) {
if ( this == o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
index fc23cb4..e498b07 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/VersionedMigrationSetTest.java
@@ -114,17 +114,13 @@ public class VersionedMigrationSetTest {
new MigrationRelationship<>( new TestVersionImpl( 4 ), new TestVersionImpl( 6 ) );
- final MigrationRelationship<TestVersionImpl> relationship5_6 =
- new MigrationRelationship<>( new TestVersionImpl( 5 ), new TestVersionImpl( 6 ) );
-
-
final MigrationRelationship<TestVersionImpl> current =
new MigrationRelationship<>( new TestVersionImpl( 6 ), new TestVersionImpl( 6 ) );
final VersionedMigrationSet<TestVersionImpl> set =
new VersionedMigrationSet<>( relationship1_3, relationship2_3, relationship3_6, relationship4_6,
- relationship5_6, current );
+ current );
try {
@@ -156,7 +152,7 @@ public class VersionedMigrationSetTest {
migrationRelationship = set.getMigrationRelationship( 5 );
- assertEquals( relationship5_6, migrationRelationship );
+ assertEquals( relationship4_6, migrationRelationship );
migrationRelationship = set.getMigrationRelationship( 6 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
index a371340..39014ab 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
@@ -22,11 +22,14 @@ package org.apache.usergrid.persistence.model.entity;
import java.io.Serializable;
import java.util.UUID;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
/**
* Interface for creating identifiers for an entity. The implementation should implement
* the equals and hasCode methods
* @author tnine */
+@JsonDeserialize(as = SimpleId.class)
public interface Id extends Comparable<Id>, Serializable {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
index 7ad4ab4..018e90b 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
@@ -25,16 +25,29 @@ import java.util.UUID;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.model.util.Verify;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.uuid.UUIDComparator;
import com.google.common.base.Preconditions;
/** @author tnine */
+@JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
public class SimpleId implements Id, Serializable {
- private final UUID uuid;
- private final String type;
+ private UUID uuid;
+ private String type;
+
+
+ /**
+ * Do not delete! Needed for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public SimpleId(){
+
+ }
+
+
public SimpleId( final UUID uuid, final String type ) {
Preconditions.checkNotNull( uuid, "uuid is required" );
@@ -67,6 +80,26 @@ public class SimpleId implements Id, Serializable {
}
+
+ /**
+ * Do not delete! Needed for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public void setType( final String type ) {
+ this.type = type;
+ }
+
+
+
+ /**
+ * Do not delete! Needed for Jackson
+ */
+ @SuppressWarnings( "unused" )
+ public void setUuid( final UUID uuid ) {
+ this.uuid = uuid;
+ }
+
+
@Override
public boolean equals( final Object o ) {
if ( this == o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
index 1a60026..ad9b91f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
@@ -60,20 +60,12 @@ public class IndexOperationMessage implements Serializable {
}
- public void addAllIndexRequest( final Set<IndexRequest> indexRequests ) {
- this.indexRequests.addAll( indexRequests );
- }
-
public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
this.deIndexRequests.add( deIndexRequest );
}
- public void addAllDeIndexRequest( final Set<DeIndexRequest> deIndexRequests ) {
- this.deIndexRequests.addAll( deIndexRequests );
- }
-
public Set<IndexRequest> getIndexRequests() {
return indexRequests;
[48/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/01416233
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/01416233
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/01416233
Branch: refs/heads/USERGRID-593
Commit: 01416233559de620b421308494d22cdc21c9a89f
Parents: 24e7cda 34fd9cf
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Apr 23 09:31:56 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Apr 23 09:31:56 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 8 +-
.../corepersistence/CpEntityManager.java | 6 +-
.../corepersistence/CpEntityManagerFactory.java | 6 +-
.../corepersistence/CpRelationManager.java | 6 +-
.../index/AsyncIndexProvider.java | 24 +-
.../index/AsyncIndexService.java | 42 +++
.../index/AsyncReIndexService.java | 42 ---
.../index/InMemoryAsyncIndexService.java | 88 +++++
.../index/InMemoryAsyncReIndexService.java | 108 ------
.../index/IndexProcessorFig.java | 86 +++++
.../corepersistence/index/IndexService.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 9 +-
.../corepersistence/index/QueryFig.java | 103 ------
.../index/ReIndexServiceImpl.java | 12 +-
.../index/SQSAsyncIndexService.java | 325 +++++++++++++++++++
.../index/SQSAsyncReIndexService.java | 269 ---------------
.../corepersistence/util/CpNamingUtils.java | 8 +-
.../util/SerializableMapper.java | 4 +-
.../index/AsyncIndexServiceTest.java | 203 ++++++++++++
.../index/InMemoryAsycIndexServiceTest.java | 64 ++++
.../corepersistence/index/IndexServiceTest.java | 233 ++++++++++++-
.../index/SQSAsyncIndexServiceTest.java | 143 +++-----
.../impl/migration/EntityIdScope.java | 40 ++-
.../core/metrics/MetricsFactoryImpl.java | 44 +--
.../migration/data/MigrationRelationship.java | 29 +-
.../migration/data/VersionedMigrationSet.java | 31 +-
.../persistence/core/rx/RxSchedulerFig.java | 4 +-
.../core/scope/ApplicationScope.java | 4 +
.../core/scope/ApplicationScopeImpl.java | 21 +-
.../data/VersionedMigrationSetTest.java | 8 +-
.../usergrid/persistence/model/entity/Id.java | 3 +
.../persistence/model/entity/SimpleId.java | 42 ++-
.../index/impl/IndexOperationMessage.java | 8 -
33 files changed, 1259 insertions(+), 770 deletions(-)
----------------------------------------------------------------------
[37/52] [abbrv] incubator-usergrid git commit: null reference
Posted by to...@apache.org.
null reference
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a3cacb6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a3cacb6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a3cacb6a
Branch: refs/heads/USERGRID-593
Commit: a3cacb6a30db6abc7d9b9a04a2c4e819db9c2f96
Parents: c77519e
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 14:36:29 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 14:36:29 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/index/PublishRxTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a3cacb6a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
index 4e65110..973a42d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
@@ -38,7 +38,7 @@ import static org.junit.Assert.assertTrue;
/**
* Test to test some assumptions about RX behaviors
*/
-public class PublishRxtest {
+public class PublishRxTest {
@Test
public void testPublish() throws InterruptedException {
[05/52] [abbrv] incubator-usergrid git commit: add index refresh logic
Posted by to...@apache.org.
add index refresh logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/488fa936
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/488fa936
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/488fa936
Branch: refs/heads/USERGRID-593
Commit: 488fa936907a1d26d529e91761804556ebb5be46
Parents: 8d12b39
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 12:04:39 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 12:04:39 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/CandidateResults.java | 9 +--------
.../index/impl/EsApplicationEntityIndexImpl.java | 3 +--
.../persistence/index/impl/IndexRefreshCommandImpl.java | 2 ++
.../queryindex/src/test/resources/dynamic-test.properties | 2 +-
4 files changed, 5 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/488fa936/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 0e2bddf..f0955ea 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -49,16 +49,12 @@ public class CandidateResults implements Iterable<CandidateResult> {
private final List<CandidateResult> candidates;
private final Collection<SelectFieldMapping> getFieldMappings;
- private final String elasticSearchCursor;
public CandidateResults( List<CandidateResult> candidates,
- final Collection<SelectFieldMapping> getFieldMappings,
- String elasticSearchCursor
- ) {
+ final Collection<SelectFieldMapping> getFieldMappings ) {
this.candidates = candidates;
this.getFieldMappings = getFieldMappings;
- this.elasticSearchCursor = elasticSearchCursor;
}
@@ -113,7 +109,4 @@ public class CandidateResults implements Iterable<CandidateResult> {
return candidates.iterator();
}
- public String getElasticSearchCursor() {
- return elasticSearchCursor;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/488fa936/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 83e1648..6574985 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -310,10 +310,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
candidates.add( candidateResult );
}
+ final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings());
final String esScrollCursor = searchResponse.getScrollId();
- final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor );
-
// >= seems odd. However if we get an overflow, we need to account for it.
if ( esScrollCursor != null && length >= limit ) {
final String cursor = candidateResults.initializeCursor();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/488fa936/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index b916d58..804de51 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -147,6 +147,8 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
return future.doOnNext( found -> {
if ( !found.hasFinished() ) {
logger.error(String.format("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ }else{
+ logger.info(String.format("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
}
} ).doOnCompleted(() -> {
//clean up our data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/488fa936/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
index 3d4582f..9bc19ed 100644
--- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
@@ -9,7 +9,7 @@ collections.keyspace=Usergrid_Collections
cassandra.timeout=5000
index.query.limit.default=10
-elasticsearch.indexname=Usergrid
+elasticsearch.indexname=QueryIndexTests
elasticsearch.startup=external
elasticsearch.force-refresh=false
[33/52] [abbrv] incubator-usergrid git commit: Adds Apache Header and
restores content
Posted by to...@apache.org.
Adds Apache Header and restores content
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a58ccfa0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a58ccfa0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a58ccfa0
Branch: refs/heads/USERGRID-593
Commit: a58ccfa03d198bf59425bb9d0d2b78bb3b38a3f6
Parents: 28a8590
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Apr 21 14:17:26 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Apr 21 14:17:47 2015 -0600
----------------------------------------------------------------------
.../corepersistence/index/PublishRxTest.java | 92 ++++++++++++++++++-
.../corepersistence/index/PublishRxtest.java | 94 --------------------
2 files changed, 90 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a58ccfa0/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
index 1d41286..973a42d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
@@ -1,7 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.usergrid.corepersistence.index;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import rx.Observable;
+import rx.Subscription;
+import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
/**
- * Created by ApigeeCorporation on 4/21/15.
+ * Test to test some assumptions about RX behaviors
*/
-public class PublishRxTest {}
+public class PublishRxTest {
+
+ @Test
+ public void testPublish() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final Subscription connectedObservable =
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ .subscribe();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();
+
+ assertTrue( "Subscription complete", completedSubscription );
+ }
+
+
+ @Test
+ @Ignore("This seems like it should work, yet blocks forever")
+ public void testConnectableObserver() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final ConnectableObservable<Integer> connectedObservable = Observable.range( 0, count ).publish();
+
+
+ //connect to our latch, which should run on it's own subscription
+ //start our latch running
+ connectedObservable.doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ final Observable<Integer> countObservable = connectedObservable.subscribeOn( Schedulers.io() ).count();
+
+ //start the sequence
+ connectedObservable.connect();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final int returnedCount = countObservable.toBlocking().last();
+
+ assertEquals( "Counts the same", count, returnedCount );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a58ccfa0/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
deleted file mode 100644
index 6a49e80..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Ignore;
-import org.junit.Test;
-
-import rx.Observable;
-import rx.Subscription;
-import rx.observables.ConnectableObservable;
-import rx.schedulers.Schedulers;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-/**
- * Test to test some assumptions about RX behaviors
- */
-public class PublishRxTest {
-
- @Test
- public void testPublish() throws InterruptedException {
-
- final int count = 10;
-
- final CountDownLatch latch = new CountDownLatch( count );
-
- final Subscription connectedObservable =
- Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
- .subscribe();
-
-
- final boolean completed = latch.await( 5, TimeUnit.SECONDS );
-
- assertTrue( "publish1 behaves as expected", completed );
-
- final boolean completedSubscription = connectedObservable.isUnsubscribed();
-
- assertTrue( "Subscription complete", completedSubscription );
- }
-
-
- @Test
- @Ignore("This seems like it should work, yet blocks forever")
- public void testConnectableObserver() throws InterruptedException {
-
- final int count = 10;
-
- final CountDownLatch latch = new CountDownLatch( count );
-
- final ConnectableObservable<Integer> connectedObservable = Observable.range( 0, count ).publish();
-
-
- //connect to our latch, which should run on it's own subscription
- //start our latch running
- connectedObservable.doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() ).subscribe();
-
-
- final Observable<Integer> countObservable = connectedObservable.subscribeOn( Schedulers.io() ).count();
-
- //start the sequence
- connectedObservable.connect();
-
-
- final boolean completed = latch.await( 5, TimeUnit.SECONDS );
-
- assertTrue( "publish1 behaves as expected", completed );
-
- final int returnedCount = countObservable.toBlocking().last();
-
- assertEquals( "Counts the same", count, returnedCount );
- }
-}
[32/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/28a8590c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/28a8590c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/28a8590c
Branch: refs/heads/USERGRID-593
Commit: 28a8590cd3c9db202a858df2b5605d7d48dc29f4
Parents: 5f99ee2 f2aa403
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Apr 21 14:15:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Apr 21 14:15:25 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/index/PublishRxTest.java | 7 +++++++
.../apache/usergrid/corepersistence/index/PublishRxtest.java | 4 +---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/28a8590c/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
index 84d3ef7,0000000..6a49e80
mode 100644,000000..100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
@@@ -1,96 -1,0 +1,94 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
-
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import rx.Observable;
+import rx.Subscription;
+import rx.observables.ConnectableObservable;
+import rx.schedulers.Schedulers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+/**
- * Validates the Rx scheduler works as expected with publish
++ * Test to test some assumptions about RX behaviors
+ */
+public class PublishRxTest {
+
+ @Test
+ public void testPublish() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final Subscription connectedObservable =
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ .subscribe();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();
- ;
+
+ assertTrue( "Subscription complete", completedSubscription );
+ }
+
+
+ @Test
+ @Ignore("This seems like it should work, yet blocks forever")
+ public void testConnectableObserver() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final ConnectableObservable<Integer> connectedObservable = Observable.range( 0, count ).publish();
+
+
+ //connect to our latch, which should run on it's own subscription
+ //start our latch running
+ connectedObservable.doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ final Observable<Integer> countObservable = connectedObservable.subscribeOn( Schedulers.io() ).count();
+
+ //start the sequence
+ connectedObservable.connect();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final int returnedCount = countObservable.toBlocking().last();
+
+ assertEquals( "Counts the same", count, returnedCount );
+ }
+}
[25/52] [abbrv] incubator-usergrid git commit: scroll changes
Posted by to...@apache.org.
scroll changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a2d64a76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a2d64a76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a2d64a76
Branch: refs/heads/USERGRID-593
Commit: a2d64a76291b5274508aa21a6003923c581a7cbd
Parents: f271496
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 09:01:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 09:01:59 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsApplicationEntityIndexImpl.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a2d64a76/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 05318a2..3a28549 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -294,6 +294,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
// >= seems odd. However if we get an overflow, we need to account for it.
if ( length >= limit ) {
+
final String cursor = candidateResults.initializeCursor();
//now set this into our map module
[26/52] [abbrv] incubator-usergrid git commit: change cursor to offset
Posted by to...@apache.org.
change cursor to offset
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f130b7ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f130b7ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f130b7ce
Branch: refs/heads/USERGRID-593
Commit: f130b7ceaf1cfcaa9737ba4ddebfa42ba04581b1
Parents: a2d64a7
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 09:50:52 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 09:50:52 2015 -0600
----------------------------------------------------------------------
.../index/ApplicationEntityIndex.java | 11 +-
.../persistence/index/CandidateResults.java | 30 ++---
.../impl/EsApplicationEntityIndexImpl.java | 133 +------------------
.../persistence/index/impl/EntityIndexTest.java | 15 +--
4 files changed, 25 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f130b7ce/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index b06cefe..e9374c4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -45,16 +45,7 @@ public interface ApplicationEntityIndex {
final int limit );
CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
- final int limit, final int from );
-
- /**
- * Get next page of results from a previous cursor. Note that limit used here should be the same limit as the initial
- * Cursor. Failure to do so can result in strange cursor behavior on the response.
- *
- * @param cursor The cursor from the original search
- * @return The next page of candidate results
- */
- CandidateResults getNextPage( final String cursor);
+ final int limit, final int offset );
/**
* delete all application records
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f130b7ce/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 5dc5502..716c7d8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -27,13 +27,9 @@ package org.apache.usergrid.persistence.index;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import com.google.common.base.Optional;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
/**
@@ -42,9 +38,7 @@ import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUU
*/
public class CandidateResults implements Iterable<CandidateResult> {
- private static final Logger log = LoggerFactory.getLogger( CandidateResults.class );
-
- private String cursor = null;
+ private Optional<Integer> offset = null;
private final List<CandidateResult> candidates;
@@ -56,27 +50,27 @@ public class CandidateResults implements Iterable<CandidateResult> {
) {
this.candidates = candidates;
this.getFieldMappings = getFieldMappings;
+ offset = Optional.absent();
}
- public String initializeCursor(){
- cursor = sanitizeUUID( UUIDGenerator.newTimeUUID() );
- return cursor;
+ public void initializeCursor(int offset){
+ this.offset = Optional.of(offset);
}
- public boolean hasCursor() {
- return cursor != null;
+ public boolean hasOffset() {
+ return offset.isPresent();
}
- public String getCursor() {
- return cursor;
+ public Optional<Integer> getOffset() {
+ return offset;
}
- public void setCursor(String cursor) {
- this.cursor = cursor;
+ public void setOffset(int offset) {
+ this.offset = Optional.of(offset);
}
@@ -102,7 +96,7 @@ public class CandidateResults implements Iterable<CandidateResult> {
* @return
*/
public CandidateResult get(int index){
- return candidates.get( index );
+ return candidates.get(index);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f130b7ce/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 3a28549..672f090 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -153,7 +153,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
}
public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
- final int limit, final int from ) {
+ final int limit, final int offset ) {
IndexValidationUtils.validateSearchEdge( searchEdge );
Preconditions.checkNotNull( searchTypes, "searchTypes cannot be null" );
@@ -165,7 +165,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final ParsedQuery parsedQuery = ParsedQueryBuilder.build( query );
- final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, from );
+ final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, offset );
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n nodeId: {}, edgeType: {}, \n type: {}\n query: {} ",
@@ -186,34 +186,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
}
failureMonitor.success();
- return parseResults( searchResponse, searchEdge, searchTypes, parsedQuery, limit, from );
- }
-
-
- public CandidateResults getNextPage( final String cursor ) {
- Preconditions.checkNotNull(cursor, "cursor is a required argument");
-
- String userCursorString = cursor;
-
- //sanitiztion, we probably shouldn't do this here, but in the caller
- if ( userCursorString.startsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 1 );
- }
- if ( userCursorString.endsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
- }
-
- //now get the cursor from the map and validate
- final String queryStateString = mapManager.getString( userCursorString );
-
-
- if(queryStateString == null){
- throw new QueryException( String.format("Could not find a cursor for the value '%s' ", userCursorString ));
- }
-
- //parse the query state
- final QueryState queryState = QueryState.fromSerialized( queryStateString );
- return search(queryState.searchEdge,queryState.searchTypes,queryState.ql,queryState.limit,queryState.from);
+ return parseResults( searchResponse, parsedQuery, limit, offset );
}
@@ -271,8 +244,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
/**
* Parse the results and return the canddiate results
*/
- private CandidateResults parseResults( final SearchResponse searchResponse, final SearchEdge searchEdge,
- final SearchTypes searchTypes, final ParsedQuery query,
+ private CandidateResults parseResults( final SearchResponse searchResponse, final ParsedQuery query,
final int limit, final int from ) {
final SearchHits searchHits = searchResponse.getHits();
@@ -295,106 +267,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
// >= seems odd. However if we get an overflow, we need to account for it.
if ( length >= limit ) {
- final String cursor = candidateResults.initializeCursor();
-
- //now set this into our map module
- final int minutes = indexFig.getQueryCursorTimeout();
- //just truncate it, we'll never hit a long value anyway
-
- final int storageSeconds = ( int ) TimeUnit.MINUTES.toSeconds( minutes );
-
- final QueryState state = new QueryState( query.getOriginalQuery(), searchEdge, searchTypes, limit, from );
-
- final String queryStateSerialized = state.serialize();
-
- mapManager.putString(cursor, queryStateSerialized, storageSeconds);
+ candidateResults.initializeCursor(from);
}
return candidateResults;
}
-
- /**
- * Class to encapsulate our serialized state
- */
- private static final class QueryState implements Serializable{
-
-
- private static final JsonFactory SMILE_FACTORY = new JsonFactory();
-
- private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
-
- /**
- * Our reserved character for constructing our storage string
- */
- private static final String STORAGE_DELIM = "_ugdelim_";
-
- private final String ql;
-
- private final SearchEdge searchEdge;
- private final SearchTypes searchTypes;
- private final int limit;
- private final int from;
-
-
- private QueryState( final String ql,final SearchEdge searchEdge, final SearchTypes searchTypes, final int limit, final int from ) {
- this.ql = ql;
- this.searchEdge = searchEdge;
- this.searchTypes = searchTypes;
- this.limit = limit;
- this.from = from;
- }
-
-
- /**
- * Factory to create an instance of our state from the serialized string
- */
- public static QueryState fromSerialized( final String input ) {
-
- final String[] parts = input.split(STORAGE_DELIM);
-
- Preconditions.checkArgument(parts != null && parts.length == 7,
- "there must be 5 parts to the serialized query state");
- try {
-
- String edgeName = parts[1];
- Id edgeId = MAPPER.readValue(parts[2],SimpleId.class);
- SearchEdge.NodeType nodeType = SearchEdge.NodeType.valueOf(parts[3]);
-
- String[] searchTypes = MAPPER.readValue(parts[4],String[].class);
- SearchTypes types = searchTypes.length == 0 ? SearchTypes.allTypes() : SearchTypes.fromTypes(searchTypes);
- return new QueryState(parts[0], new SearchEdgeImpl(edgeId,edgeName,nodeType),types , Integer.parseInt(parts[5]), Integer.parseInt(parts[6]));
- } catch (Exception jpe) {
- throw new RuntimeException(jpe);
- }
- }
-
-
- public String serialize() {
- try {
-
- StringBuilder storageString = new StringBuilder();
-
- storageString.append(ql).append(STORAGE_DELIM);
-
- String edge =searchEdge.getEdgeName();
- storageString.append(edge).append(STORAGE_DELIM);
- String nodeid = MAPPER.writeValueAsString(searchEdge.getNodeId());
- storageString.append(nodeid).append(STORAGE_DELIM);
- String nodeType = searchEdge.getNodeType().name();
- storageString.append(nodeType).append(STORAGE_DELIM);
- String types = MAPPER.writeValueAsString(searchTypes.getTypes());
- storageString.append(types).append(STORAGE_DELIM);
-
- storageString.append(limit).append(STORAGE_DELIM);
-
- storageString.append(from+limit);
-
- return storageString.toString();
- }catch (Exception jpe){
- throw new RuntimeException(jpe);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f130b7ce/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index cd61818..f0a1630 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -28,6 +28,7 @@ import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Optional;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -699,23 +700,21 @@ public class EntityIndexTest extends BaseIT {
final int expectedPages = size / limit;
- String cursor = null;
+ Optional<Integer> offset = Optional.absent();
UUID lastId = null;
final String query = "select * where mytype='"+myType+"' order by ordinal asc";
int i ;
for ( i=0; i < expectedPages; i++ ) {
- //**
-
- final CandidateResults results = cursor == null
- ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, i* limit )
- : entityIndex.getNextPage(cursor);
+ final CandidateResults results = !offset.isPresent()
+ ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit )
+ : entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit);
assertEquals(limit, results.size());
int ordinal = 0;//i == 0 ? 0 : 1;
assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
lastId = results.get(limit -1).getId().getUuid();
- cursor = results.getCursor();
+ offset = results.getOffset();
assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) );
}
@@ -723,7 +722,7 @@ public class EntityIndexTest extends BaseIT {
final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit);
assertEquals( 0, results.size() );
- assertNull( results.getCursor() );
+ assertFalse(results.hasOffset());
}
[49/52] [abbrv] incubator-usergrid git commit: Updated RX java to
1.0.9 Updates flush logic. Buffers of size > 1 are not getting flushed
correctly. Still has bugs
Posted by to...@apache.org.
Updated RX java to 1.0.9
Updates flush logic. Buffers of size > 1 are not getting flushed correctly. Still has bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e91930ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e91930ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e91930ff
Branch: refs/heads/USERGRID-593
Commit: e91930ff0679ff3709490aacfdaec67e1ba83301
Parents: 34fd9cf
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Apr 23 10:24:20 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Apr 23 10:24:20 2015 -0600
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceTest.java | 44 +++++---
stack/core/src/test/resources/log4j.properties | 1 +
stack/corepersistence/pom.xml | 2 +-
.../usergrid/persistence/index/IndexFig.java | 18 ++-
.../persistence/index/impl/BatchOperation.java | 41 +++++++
.../persistence/index/impl/BatchRequest.java | 41 -------
.../index/impl/DeIndexOperation.java | 104 +++++++++++++++++
.../persistence/index/impl/DeIndexRequest.java | 104 -----------------
.../index/impl/EsEntityIndexBatchImpl.java | 4 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 107 ++++++++++--------
.../persistence/index/impl/IndexOperation.java | 107 ++++++++++++++++++
.../index/impl/IndexOperationMessage.java | 19 ++--
.../index/impl/IndexRefreshCommandImpl.java | 6 +-
.../persistence/index/impl/IndexRequest.java | 111 -------------------
14 files changed, 371 insertions(+), 338 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index 8be93f6..92eb2e0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -20,9 +20,9 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.List;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
import org.junit.Before;
import org.junit.Test;
@@ -46,7 +46,7 @@ import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.EsRunner;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.index.impl.IndexOperation;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -117,10 +117,10 @@ public class IndexServiceTest {
//real users should never call to blocking, we're not sure what we'll get
final IndexOperationMessage results = indexed.toBlocking().last();
- final Set<IndexRequest> indexRequests = results.getIndexRequests();
+ final Set<IndexOperation> indexRequests = results.getIndexRequests();
//ensure our value made it to the index request
- final IndexRequest indexRequest = indexRequests.iterator().next();
+ final IndexOperation indexRequest = indexRequests.iterator().next();
assertNotNull( indexRequest );
}
@@ -175,9 +175,6 @@ public class IndexServiceTest {
final SearchEdge collectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( collectionEdge );
- final CountDownLatch latch = new CountDownLatch( 1 );
-
-
//query until it's available
final CandidateResults collectionResults = getResults( applicationEntityIndex, collectionSearchEdge,
SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
@@ -186,7 +183,6 @@ public class IndexServiceTest {
assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
- latch.await();
final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
@@ -235,14 +231,23 @@ public class IndexServiceTest {
* Write 10k edges 10 at a time in parallel
*/
- final int edgeCount = 2000;
+// final int edgeCount = indexFig.getIndexBatchSize()*2;
+ final int edgeCount = 2;
- final Edge connectionSearch = Observable.range( 0, edgeCount ).flatMap( integer -> {
+ final List<Edge> connectionSearchEdges = Observable.range( 0, edgeCount ).flatMap( integer -> {
final Id connectingId = createId( "connecting" );
final Edge edge = CpNamingUtils.createConnectionEdge( connectingId, "likes", testEntity.getId() );
return graphManager.writeEdge( edge ).subscribeOn( Schedulers.io() );
- }, 10 ).toBlocking().last();
+ }, 20).toList().toBlocking().last();
+
+
+ assertEquals( "All edges saved", edgeCount, connectionSearchEdges.size() );
+
+ //get the first and last edge
+ final Edge connectionSearch = connectionSearchEdges.get( 0 );
+
+ final Edge lastSearch = connectionSearchEdges.get( edgeCount-1 );
//now index
@@ -258,8 +263,6 @@ public class IndexServiceTest {
final SearchEdge collectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( collectionEdge );
- final CountDownLatch latch = new CountDownLatch( 1 );
-
//query until it's available
final CandidateResults collectionResults = getResults( applicationEntityIndex, collectionSearchEdge,
@@ -269,7 +272,6 @@ public class IndexServiceTest {
assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
- latch.await();
final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
@@ -281,6 +283,18 @@ public class IndexServiceTest {
assertEquals( 1, connectionResults.size() );
assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
+
+
+ final SearchEdge lastConnectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( lastSearch );
+
+
+ //query until it's available
+ final CandidateResults lastConnectionResults = getResults( applicationEntityIndex, lastConnectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, lastConnectionResults.size() );
+
+ assertEquals( testEntity.getId(), lastConnectionResults.get( 0 ).getId() );
}
@@ -291,7 +305,7 @@ public class IndexServiceTest {
for ( int i = 0; i < attempts; i++ ) {
final CandidateResults candidateResults =
- applicationEntityIndex.search( searchEdge, searchTypes, "select *", 100 );
+ applicationEntityIndex.search( searchEdge, searchTypes, ql , count );
if ( candidateResults.size() == expectedSize ) {
return candidateResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index acab352..34bd708 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -64,4 +64,5 @@ log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndex
#log4j.logger.org.elasticsearch=DEBUG
#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
+log4j.logger.org.apache.usergrid.corepersistence.index.IndexServiceImpl=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 8494996..8bd8613 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -71,7 +71,7 @@ limitations under the License.
<junit.version>4.11</junit.version>
<kryo-serializers.version>0.26</kryo-serializers.version>
<log4j.version>1.2.17</log4j.version>
- <rx.version>1.0.8</rx.version>
+ <rx.version>1.0.9</rx.version>
<slf4j.version>1.7.2</slf4j.version>
<surefire.version>2.16</surefire.version>
<aws.version>1.9.0</aws.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 47b25ae..eb59625 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -61,23 +61,27 @@ public interface IndexFig extends GuicyFig {
String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
/**
+ * The number of worker threads to flush collapsed batches
+ */
+ String INDEX_FLUSH_WORKER_COUNT = "index.flush.workers";
+
+ /**
* the number of times we can fail before we refresh the client
*/
String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
+ String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
- String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
-
/**
* Timeout calls to elasticsearch.
* @return
*/
- public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+ String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
- public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
+ String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
/**
* The client type to use. Valid values are NODE or TRANSPORT
@@ -156,6 +160,10 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_BUFFER_SIZE )
int getIndexBufferSize();
+ @Default("10")
+ @Key(INDEX_FLUSH_WORKER_COUNT)
+ int getIndexFlushWorkerCount();
+
/**
@@ -184,7 +192,7 @@ public interface IndexFig extends GuicyFig {
@Key("elasticsearch.refresh_sleep_ms")
@Default("200")
long refreshSleep();
-
+
@Default( "5000" )
@Key( ELASTICSEARCH_QUERY_TIMEOUT )
long getQueryTimeout();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchOperation.java
new file mode 100644
index 0000000..117532a
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchOperation.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.io.Serializable;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.client.Client;
+
+
+/**
+ * A batch request we can serialize and construct on receive
+ */
+public interface BatchOperation extends Serializable {
+
+
+ /**
+ * Passing the client and the bulk request, add ourselves to the bulk request
+ * @param client
+ * @param bulkRequest
+ */
+ void doOperation( final Client client, final BulkRequestBuilder bulkRequest );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
deleted file mode 100644
index df6c5b0..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BatchRequest.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.io.Serializable;
-
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.client.Client;
-
-
-/**
- * A batch request we can serialize and construct on receive
- */
-public interface BatchRequest extends Serializable {
-
-
- /**
- * Passing the client and the bulk request, add ourselves to the bulk request
- * @param client
- * @param bulkRequest
- */
- public void doOperation(final Client client, final BulkRequestBuilder bulkRequest );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
new file mode 100644
index 0000000..4f47749
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.Arrays;
+import java.util.UUID;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.delete.DeleteRequestBuilder;
+import org.elasticsearch.client.Client;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+
+
+/**
+ * Represent the properties required to build an index request
+ */
+@JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" )
+public class DeIndexOperation implements BatchOperation {
+
+ private String[] indexes;
+ private String documentId;
+
+
+ public DeIndexOperation() {
+
+ }
+
+
+ public DeIndexOperation( String[] indexes, ApplicationScope applicationScope, SearchEdge searchEdge, Id id,
+ UUID version ) {
+ this.indexes = indexes;
+ this.documentId = createIndexDocId( applicationScope, id, version, searchEdge );
+ }
+
+
+ @Override
+ public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
+
+
+ for ( final String index : indexes ) {
+ final DeleteRequestBuilder builder =
+ client.prepareDelete( index, IndexingUtils.ES_ENTITY_TYPE, documentId );
+ bulkRequest.add( builder );
+ }
+ }
+
+
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final DeIndexOperation that = ( DeIndexOperation ) o;
+
+ if ( !documentId.equals( that.documentId ) ) {
+ return false;
+ }
+
+ if ( !Arrays.equals( indexes, that.indexes ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = Arrays.hashCode( indexes );
+ result = 31 * result + documentId.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
deleted file mode 100644
index 3f0dc23..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.Arrays;
-import java.util.UUID;
-
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.client.Client;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
-
-
-/**
- * Represent the properties required to build an index request
- */
-@JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" )
-public class DeIndexRequest implements BatchRequest {
-
- private String[] indexes;
- private String documentId;
-
-
- public DeIndexRequest() {
-
- }
-
-
- public DeIndexRequest( String[] indexes, ApplicationScope applicationScope, SearchEdge searchEdge, Id id,
- UUID version ) {
- this.indexes = indexes;
- this.documentId = createIndexDocId( applicationScope, id, version, searchEdge );
- }
-
-
- @Override
- public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
-
-
- for ( final String index : indexes ) {
- final DeleteRequestBuilder builder =
- client.prepareDelete( index, IndexingUtils.ES_ENTITY_TYPE, documentId );
- bulkRequest.add( builder );
- }
- }
-
-
-
-
- @Override
- public boolean equals( final Object o ) {
- if ( this == o ) {
- return true;
- }
- if ( o == null || getClass() != o.getClass() ) {
- return false;
- }
-
- final DeIndexRequest that = ( DeIndexRequest ) o;
-
- if ( !documentId.equals( that.documentId ) ) {
- return false;
- }
-
- if ( !Arrays.equals( indexes, that.indexes ) ) {
- return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode() {
- int result = Arrays.hashCode( indexes );
- result = 31 * result + documentId.hashCode();
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 32a0f02..66849ee 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -81,7 +81,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
//add app id for indexing
- container.addIndexRequest( new IndexRequest( writeAlias, applicationScope, indexEdge, entity ) );
+ container.addIndexRequest( new IndexOperation( writeAlias, applicationScope, indexEdge, entity ) );
return this;
}
@@ -105,7 +105,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- container.addDeIndexRequest( new DeIndexRequest( indexes, applicationScope, searchEdge, id, version ) );
+ container.addDeIndexRequest( new DeIndexOperation( indexes, applicationScope, searchEdge, id, version ) );
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 7cc7dcf..0577f28 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -124,67 +125,80 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private void startSubscription() {
- Observable.create( bufferProducer )
- .buffer( indexFig.getIndexBufferSize(), indexFig.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
- Schedulers.newThread() )
+ final Observable<IndexOperationMessage> observable = Observable.create( bufferProducer );
- .doOnNext( containerList -> {
- if ( containerList.size() == 0 ) {
- return;
- }
+ //buffer on our new thread with a timeout
+ observable.buffer( indexFig.getIndexBufferSize(), indexFig.getIndexBufferTimeout(), TimeUnit.MILLISECONDS,
+ Schedulers.newThread() ).flatMap( indexOpBuffer -> {
- flushMeter.mark( containerList.size() );
- Timer.Context time = flushTimer.time();
+ //hand off to processor in new observable thread so we can continue to buffer faster
+ return Observable.just( indexOpBuffer ).flatMap(
+ indexOpBufferObservable -> processBatch( indexOpBufferObservable ) )
+ //use the I/O scheduler for thread re-use and efficiency in context switching then use our concurrent
+ // flatmap count or higher throughput of batches once buffered
+ .subscribeOn( Schedulers.io() );
+ }, indexFig.getIndexFlushWorkerCount() )
+ //start in the background
+ .subscribe();
+ }
- execute( containerList );
- time.stop();
- } )
- //ack after we process
- .doOnNext( indexOperationMessages -> {
+ /**
+ * Process the buffer of batches
+ * @param batches
+ * @return
+ */
+ private Observable<IndexOperationMessage> processBatch( final List<IndexOperationMessage> batches ) {
- //release so we know we've done processing
- inFlight.addAndGet( -1 * indexOperationMessages.size() );
- } ).subscribe();
- //start in the background
+ final Observable<IndexOperationMessage> indexOps = Observable.from( batches );
- }
+ //take our stream of batches, then stream then into individual ops for consumption on ES
+ final Observable<BatchOperation> batchOps = indexOps.flatMap( batch -> {
+ final Set<IndexOperation> indexOperationSet = batch.getIndexRequests();
+ final Set<DeIndexOperation> deIndexOperationSet = batch.getDeIndexRequests();
- /**
- * Execute the request, check for errors, then re-init the batch for future use
- */
- private void execute( final List<IndexOperationMessage> operationMessages ) {
+ final int indexOperationSetSize = indexOperationSet.size();
+ final int deIndexOperationSetSize = deIndexOperationSet.size();
- if ( operationMessages == null || operationMessages.size() == 0 ) {
- return;
- }
+ log.debug( "Emitting {} add and {} remove operations", indexOperationSetSize, deIndexOperationSetSize );
- //process and flatten all the messages to builder requests
- //batch shard operations into a bulk request
- Observable.from( operationMessages ).flatMap( indexOperationMessage -> {
- final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
- final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ indexSizeCounter.dec( indexOperationSetSize );
+ indexSizeCounter.dec( deIndexOperationSetSize );
- indexSizeCounter.dec( indexOperationMessage.getDeIndexRequests().size() );
- indexSizeCounter.dec( indexOperationMessage.getIndexRequests().size() );
+ final Observable<IndexOperation> index = Observable.from( batch.getIndexRequests() );
+ final Observable<DeIndexOperation> deIndex = Observable.from( batch.getDeIndexRequests() );
return Observable.merge( index, deIndex );
- } )
- //collection all the operations into a single stream
- .collect( () -> initRequest(), ( bulkRequestBuilder, batchRequest ) -> {
- batchRequest.doOperation( client, bulkRequestBuilder );
- } ) //send the request off to ES
- .doOnNext( bulkRequestBuilder -> sendRequest( bulkRequestBuilder ) ).toBlocking().lastOrDefault( null );
-
- //call back all futures
- Observable.from( operationMessages ).doOnNext( operationMessage -> operationMessage.getFuture().done() )
- .toBlocking().lastOrDefault( null );
+ } );
+
+ //buffer into the max size we can send ES and fire them all off until we're completed
+ final Observable<BulkRequestBuilder> requests = batchOps.buffer( indexFig.getIndexBatchSize() )
+ //flatten the buffer into a single batch execution
+ .flatMap( individualOps -> Observable.from( individualOps )
+ //collect them
+ .collect( () -> initRequest(), ( bulkRequestBuilder, batchOperation ) -> {
+ log.debug( "adding operation {} to bulkRequestBuilder {}", batchOperation, bulkRequestBuilder );
+ batchOperation.doOperation( client, bulkRequestBuilder );
+ } ) )
+ //write them
+ .doOnNext( bulkRequestBuilder -> sendRequest( bulkRequestBuilder ) ).doOnError( t -> log.error( "Unable to process batches", t ) );
+
+
+ //now that we've processed them all, ack the futures after our last batch comes through
+ final Observable<IndexOperationMessage> processedIndexOperations =
+ requests.last().flatMap( lastRequest -> Observable.from( batches ) );
+
+ //subscribe to the operations that generate requests on a new thread so that we can execute them quickly
+ //mark this as done
+ return processedIndexOperations.doOnNext( processedIndexOp -> processedIndexOp.done() ).doOnError( t -> log.error( "Unable to ack futures", t ) );
}
+ /*
+
/**
* initialize request
*/
@@ -209,7 +223,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
try {
- responses = bulkRequest.execute().actionGet(indexFig.getWriteTimeout());
+ responses = bulkRequest.execute().actionGet( indexFig.getWriteTimeout() );
}
catch ( Throwable t ) {
log.error( "Unable to communicate with elasticsearch" );
@@ -249,6 +263,11 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
* Send the data through the buffer
*/
public void send( final IndexOperationMessage indexOp ) {
+ //no-op
+ if ( indexOp.isEmpty() ) {
+ indexOp.done();
+ return;
+ }
subscriber.onNext( indexOp );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
new file mode 100644
index 0000000..2e2f188
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.Map;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
+
+
+/**
+ * Represent the properties required to build an index request
+ */
+public class IndexOperation implements BatchOperation {
+
+ public String writeAlias;
+ public String documentId;
+
+ public Map<String, Object> data;
+
+ public IndexOperation( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge,
+ Entity entity ) {
+ this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge), EntityToMapConverter.convert(applicationScope,indexEdge, entity));
+
+ }
+
+ public IndexOperation( final String writeAlias, String documentId, Map<String, Object> data ) {
+ this.writeAlias = writeAlias;
+ this.data = data;
+ this.documentId = documentId;
+ }
+
+ /**
+ * DO NOT DELETE! Required for Jackson
+ */
+ public IndexOperation() {
+ }
+
+
+ public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
+ IndexRequestBuilder builder = client.prepareIndex( writeAlias, IndexingUtils.ES_ENTITY_TYPE, documentId ).setSource( data );
+
+
+ bulkRequest.add( builder );
+ }
+
+
+
+ @Override
+ public boolean equals( final Object o ) {
+ if ( this == o ) {
+ return true;
+ }
+ if ( o == null || getClass() != o.getClass() ) {
+ return false;
+ }
+
+ final IndexOperation that = ( IndexOperation ) o;
+
+ if ( !data.equals( that.data ) ) {
+ return false;
+ }
+ if ( !documentId.equals( that.documentId ) ) {
+ return false;
+ }
+
+ if ( !writeAlias.equals( that.writeAlias ) ) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ @Override
+ public int hashCode() {
+ int result = writeAlias.hashCode();
+ result = 31 * result + documentId.hashCode();
+ result = 31 * result + data.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
index ad9b91f..e830476 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperationMessage.java
@@ -34,8 +34,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
* Container for index operations.
*/
public class IndexOperationMessage implements Serializable {
- private final Set<IndexRequest> indexRequests;
- private final Set<DeIndexRequest> deIndexRequests;
+ private final Set<IndexOperation> indexRequests;
+ private final Set<DeIndexOperation> deIndexRequests;
@@ -46,33 +46,28 @@ public class IndexOperationMessage implements Serializable {
final IndexOperationMessage parent = this;
this.indexRequests = new HashSet<>();
this.deIndexRequests = new HashSet<>();
- this.containerFuture = new BetterFuture<>( new Callable<IndexOperationMessage>() {
- @Override
- public IndexOperationMessage call() throws Exception {
- return parent;
- }
- } );
+ this.containerFuture = new BetterFuture<>( () -> parent );
}
- public void addIndexRequest( final IndexRequest indexRequest ) {
+ public void addIndexRequest( final IndexOperation indexRequest ) {
indexRequests.add( indexRequest );
}
- public void addDeIndexRequest( final DeIndexRequest deIndexRequest ) {
+ public void addDeIndexRequest( final DeIndexOperation deIndexRequest ) {
this.deIndexRequests.add( deIndexRequest );
}
- public Set<IndexRequest> getIndexRequests() {
+ public Set<IndexOperation> getIndexRequests() {
return indexRequests;
}
- public Set<DeIndexRequest> getDeIndexRequests() {
+ public Set<DeIndexOperation> getDeIndexRequests() {
return deIndexRequests;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index c997e5a..3f2a56f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@ -104,7 +104,7 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
final String entityId = entityData.get( IndexingUtils.ENTITY_ID_FIELDNAME ).toString();
//add a tracer record
- IndexRequest indexRequest = new IndexRequest( alias.getWriteAlias(), docId, entityData );
+ IndexOperation indexRequest = new IndexOperation( alias.getWriteAlias(), docId, entityData );
//save the item
IndexOperationMessage message = new IndexOperationMessage();
@@ -155,8 +155,8 @@ public class IndexRefreshCommandImpl implements IndexRefreshCommand {
} ).doOnCompleted(() -> {
//clean up our data
String[] aliases = indexCache.getIndexes(alias, AliasedEntityIndex.AliasType.Read);
- DeIndexRequest deIndexRequest =
- new DeIndexRequest(aliases, appScope, edge, entity.getId(), entity.getVersion());
+ DeIndexOperation deIndexRequest =
+ new DeIndexOperation(aliases, appScope, edge, entity.getId(), entity.getVersion());
//delete the item
IndexOperationMessage indexOperationMessage =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e91930ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
deleted file mode 100644
index 5852394..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.Map;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.IndexEdge;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchType;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.client.Client;
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
-
-
-/**
- * Represent the properties required to build an index request
- */
-public class IndexRequest implements BatchRequest {
-
- public String writeAlias;
- public String documentId;
-
- public Map<String, Object> data;
-
- public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexEdge indexEdge, Entity entity) {
- this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge), EntityToMapConverter.convert(applicationScope,indexEdge, entity));
-
- }
-
- public IndexRequest( final String writeAlias, String documentId, Map<String, Object> data) {
- this.writeAlias = writeAlias;
- this.data = data;
- this.documentId = documentId;
- }
-
- /**
- * DO NOT DELETE! Required for Jackson
- */
- public IndexRequest() {
- }
-
-
- public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
- IndexRequestBuilder builder = client.prepareIndex( writeAlias, IndexingUtils.ES_ENTITY_TYPE, documentId ).setSource( data );
-
-
- bulkRequest.add( builder );
- }
-
-
-
- @Override
- public boolean equals( final Object o ) {
- if ( this == o ) {
- return true;
- }
- if ( o == null || getClass() != o.getClass() ) {
- return false;
- }
-
- final IndexRequest that = ( IndexRequest ) o;
-
- if ( !data.equals( that.data ) ) {
- return false;
- }
- if ( !documentId.equals( that.documentId ) ) {
- return false;
- }
-
- if ( !writeAlias.equals( that.writeAlias ) ) {
- return false;
- }
-
- return true;
- }
-
-
- @Override
- public int hashCode() {
- int result = writeAlias.hashCode();
- result = 31 * result + documentId.hashCode();
- result = 31 * result + data.hashCode();
- return result;
- }
-}
[51/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' into USERGRID-593
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 0000000,e5fbac9..1c28e6a
mode 000000,100644..100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@@ -1,0 -1,921 +1,912 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.usergrid.persistence;
+
+
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.io.UnsupportedEncodingException;
+ import java.net.URLDecoder;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Map.Entry;
+ import java.util.UUID;
+
+ import com.netflix.astyanax.serializers.IntegerSerializer;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import org.apache.commons.codec.binary.Base64;
+ import org.apache.commons.lang.StringUtils;
+
+ import org.apache.usergrid.persistence.index.SelectFieldMapping;
+ import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+ import org.apache.usergrid.persistence.index.query.CounterResolution;
+ import org.apache.usergrid.persistence.index.query.Identifier;
+ import org.apache.usergrid.persistence.index.query.tree.Operand;
+ import org.apache.usergrid.persistence.index.utils.ClassUtils;
+ import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+ import org.apache.usergrid.persistence.index.utils.ListUtils;
+ import org.apache.usergrid.persistence.index.utils.MapUtils;
+
+ import com.fasterxml.jackson.annotation.JsonIgnore;
+ import com.fasterxml.jackson.databind.ObjectMapper;
+ import com.google.common.base.Optional;
+
+
+ public class Query {
+ private static final Logger logger = LoggerFactory.getLogger( Query.class );
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+
+ public enum Level {
+ IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
+ }
+
+ public static final int DEFAULT_LIMIT = 10;
+
+ public static final int MAX_LIMIT = 1000;
+
+ public static final String PROPERTY_UUID = "uuid";
+
+ private String type;
+ private Operand rootOperand;
+ private UUID startResult;
+ private Optional<Integer> offset;
+ private int limit = 0;
+
+ private boolean mergeSelectResults = false;
+ private Level level = Level.ALL_PROPERTIES;
+ private String connectionType;
+ private List<String> permissions;
+ private boolean reversed;
+ private boolean reversedSet = false;
+ private Long startTime;
+ private Long finishTime;
+ private boolean pad;
+ private CounterResolution resolution = CounterResolution.ALL;
+ private List<Identifier> identifiers;
+ private List<CounterFilterPredicate> counterFilters;
+ private String collection;
+ private String ql;
+ private Collection<SelectFieldMapping> selectFields;
+
+
+ private static ObjectMapper mapper = new ObjectMapper();
+
+
+ List<Operand> filterClauses = new ArrayList<Operand>();
+
+ public Query() {
+ offset = Optional.absent();
+ }
+
+
+ /**
+ * Creates a deep copy of a query from another query
+ * @param q
+ */
+ public Query( Query q ) {
+ if ( q == null ) {
+ return;
+ }
+
+ ql = q.ql;
+ type = q.type;
+ startResult = q.startResult;
+ offset = q.offset;
+ limit = q.limit;
+ mergeSelectResults = q.mergeSelectResults;
+ //level = q.level;
+ connectionType = q.connectionType;
+ permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
+ reversed = q.reversed;
+ reversedSet = q.reversedSet;
+ startTime = q.startTime;
+ finishTime = q.finishTime;
+ resolution = q.resolution;
+ pad = q.pad;
+ rootOperand = q.rootOperand;
+ identifiers = q.identifiers != null
+ ? new ArrayList<>( q.identifiers ) : null;
+ counterFilters = q.counterFilters != null
+ ? new ArrayList<>( q.counterFilters ) : null;
+ collection = q.collection;
+ level = q.level;
+
+ }
+
+
+
+ /**
+ * Create a query instance from the QL. If the string is null, return an empty query
+ * @param ql
+ * @return
+ */
+ public static Query fromQLNullSafe(final String ql){
+ final Query query = fromQL(ql);
+
+ if(query != null){
+ return query;
+ }
+
+ return new Query();
+ }
+
+ public static Query fromQL( String ql ) throws QueryParseException {
+ if ( StringUtils.isEmpty(ql) ) {
+ return null;
+ }
+
+ Query query = new Query( );
+ query.setQl( ql );
+
+ return query;
+ }
+
+
+ /**
+ * Create a query from a property equals
+ * @param propertyName
+ * @param value
+ * @return
+ */
+ public static Query fromEquals(final String propertyName, final String value){
+ return fromQL( propertyName + " = '" + value + "'" );
+ }
+
+
+ private static Query newQueryIfNull( Query query ) {
+ if ( query == null ) {
+ query = new Query();
+ }
+ return query;
+ }
+
+
- public static Query fromJsonString( String json ) throws QueryParseException {
-
- Object o;
- try {
- o = mapper.readValue( json, Object.class );
- } catch (IOException ex) {
- throw new QueryParseException("Error parsing JSON query string " + json, ex);
- }
-
- if ( o instanceof Map ) {
- @SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, List<String>> params =
- ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
- return fromQueryParams( params );
- }
- return null;
- }
-
+
+ public static Query fromQueryParams( Map<String, List<String>> params )
+ throws QueryParseException {
+ Query q = null;
+ CounterResolution resolution = null;
+ List<Identifier> identifiers = null;
+ List<CounterFilterPredicate> counterFilters = null;
+
+ String ql = QueryUtils.queryStrFrom(params);
+ String type = ListUtils.first(params.get("type"));
+ Boolean reversed = ListUtils.firstBoolean(params.get("reversed"));
+ String connection = ListUtils.first(params.get("connectionType"));
+ UUID start = ListUtils.firstUuid(params.get("start"));
+ String cursor = ListUtils.first(params.get("cursor"));
+
+ Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
+ List<String> permissions = params.get( "permission" );
+ Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
+ Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
+
+ List<String> l = params.get( "resolution" );
+ if ( !ListUtils.isEmpty( l ) ) {
+ resolution = CounterResolution.fromString( l.get( 0 ) );
+ }
+
+ l = params.get( "counter" );
+
+ if ( !ListUtils.isEmpty( l ) ) {
+ counterFilters = CounterFilterPredicate.fromList( l );
+ }
+
+ Boolean pad = ListUtils.firstBoolean( params.get( "pad" ) );
+
+ for ( Entry<String, List<String>> param : params.entrySet() ) {
+ Identifier identifier = Identifier.from( param.getKey() );
+ if ( ( param.getValue() == null ) || ( param.getValue().size() == 0 ) || identifier.isUUID() ) {
+ if ( identifier != null ) {
+ if ( identifiers == null ) {
+ identifiers = new ArrayList<Identifier>();
+ }
+ identifiers.add( identifier );
+ }
+ }
+ }
+
+ if ( ql != null ) {
+ q = Query.fromQL( decode( ql ) );
+ }
+
+
+ if ( type != null ) {
+ q = newQueryIfNull( q );
+ q.setEntityType( type );
+ }
+
+ if ( connection != null ) {
+ q = newQueryIfNull( q );
+ q.setConnectionType( connection );
+ }
+
+ if ( permissions != null ) {
+ q = newQueryIfNull( q );
+ q.setPermissions( permissions );
+ }
+
+ if ( start != null ) {
+ q = newQueryIfNull( q );
+ q.setStartResult( start );
+ }
+
+ if ( cursor != null) {
+ q = newQueryIfNull( q );
+ q.setOffsetFromCursor(cursor);
+ }else{
+ q.offset = Optional.absent();
+ }
+
+ if ( limit != null ) {
+ q = newQueryIfNull( q );
+ q.setLimit( limit );
+ }
+
+ if ( startTime != null ) {
+ q = newQueryIfNull( q );
+ q.setStartTime( startTime );
+ }
+
+ if ( finishTime != null ) {
+ q = newQueryIfNull( q );
+ q.setFinishTime( finishTime );
+ }
+
+ if ( resolution != null ) {
+ q = newQueryIfNull( q );
+ q.setResolution( resolution );
+ }
+
+ if ( counterFilters != null ) {
+ q = newQueryIfNull( q );
+ q.setCounterFilters( counterFilters );
+ }
+
+ if ( pad != null ) {
+ q = newQueryIfNull( q );
+ q.setPad( pad );
+ }
+
+ if ( identifiers != null ) {
+ q = newQueryIfNull( q );
+ q.setIdentifiers( identifiers );
+ }
+
+ if ( reversed != null ) {
+ q = newQueryIfNull( q );
+ q.setReversed( reversed );
+ }
+
+ return q;
+ }
+
+
+ public static Query fromUUID( UUID uuid ) {
+ Query q = new Query();
+ q.addIdentifier( Identifier.fromUUID(uuid) );
+ return q;
+ }
+
+
+ public static Query fromIdentifier( Object id ) {
+ Query q = new Query();
+ q.addIdentifier( Identifier.from(id) );
+ return q;
+ }
+
+
+ public boolean hasQueryPredicates() {
+ return rootOperand != null;
+ }
+
+
+ /**
+ * Return true if the query generated select subjects
+ * @return
+ */
+ public boolean hasSelectSubjects() {
+ return this.selectFields != null || this.selectFields.size() > 0;
+ }
+
+
+ /**
+ * Set the select subjects from our query results
+ */
+ public void setSelectSubjects( final Collection<SelectFieldMapping> selectFields ) {
+ this.selectFields = selectFields;
+ }
+
+
+ /**
+ * Get the select assignments from our resetus if they were set
+ */
+ public Collection<SelectFieldMapping> getSelectAssignments() {
+ return this.selectFields;
+ }
+
+
+ public boolean containsNameOrEmailIdentifiersOnly() {
+ if ( hasQueryPredicates() ) {
+ return false;
+ }
+ if ( ( identifiers == null ) || identifiers.isEmpty() ) {
+ return false;
+ }
+ for ( Identifier identifier : identifiers ) {
+ if ( !identifier.isEmail() && !identifier.isName() ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ @JsonIgnore
+ public String getSingleNameOrEmailIdentifier() {
+ if ( !containsSingleNameOrEmailIdentifier() ) {
+ return null;
+ }
+ return ( identifiers.get( 0 ).toString() );
+ }
+
+
+ public boolean containsSingleNameOrEmailIdentifier() {
+ return containsNameOrEmailIdentifiersOnly() && ( identifiers.size() == 1 );
+ }
+
+
+ @JsonIgnore
+ public Identifier getSingleIdentifier() {
+ return identifiers != null && identifiers.size() == 1 ? identifiers.get( 0 ) : null;
+ }
+
+
+ public boolean containsSingleUuidIdentifier() {
+ return containsUuidIdentifiersOnly() && ( identifiers.size() == 1 );
+ }
+
+
+ boolean containsUuidIdentifiersOnly() {
+ if ( hasQueryPredicates() ) {
+ return false;
+ }
+ if ( ( identifiers == null ) || identifiers.isEmpty() ) {
+ return false;
+ }
+
+ for ( Identifier identifier : identifiers ) {
+ if ( !identifier.isUUID() ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ @JsonIgnore
+ public UUID getSingleUuidIdentifier() {
+ if ( !containsSingleUuidIdentifier() ) {
+ return null;
+ }
+ return ( identifiers.get( 0 ).getUUID() );
+ }
+
+
+ @JsonIgnore
+ boolean isIdsOnly() {
+ return false;
+ }
+
+ private void setIdsOnly( boolean idsOnly ) {
+ if ( idsOnly ) {
+ level = Level.IDS;
+ }
+ else if ( isIdsOnly() ) {
+ level = Level.ALL_PROPERTIES;
+ }
+ }
+
+
+ public Level getResultsLevel() {
+ isIdsOnly();
+ return level;
+ }
+
+
+ public void setResultsLevel( Level level ) {
+ setIdsOnly( level == Level.IDS );
+ this.level = level;
+ }
+
+
+ public Query withResultsLevel( Level level ) {
+ setIdsOnly( level == Level.IDS );
+ this.level = level;
+ return this;
+ }
+
+
+ public Query withReversed( boolean reversed ) {
+ setReversed( reversed );
+ return this;
+ }
+
+
+ public String getEntityType() {
+ return type;
+ }
+
+
+ public void setEntityType( String type ) {
+ this.type = type;
+ }
+
+ public List<String> getPermissions() {
+ return permissions;
+ }
+
+
+ public void setPermissions( List<String> permissions ) {
+ this.permissions = permissions;
+ }
+
+
+ public boolean isMergeSelectResults() {
+ return mergeSelectResults;
+ }
+
+
+
+ void setStartResult( UUID startResult ) {
+ this.startResult = startResult;
+ }
+
+
+ public Query withStartResult( UUID startResult ) {
+ this.startResult = startResult;
+ return this;
+ }
+
+
+ public UUID getStartResult() {
+ return startResult;
+ }
+
+
+ public Optional<Integer> getOffset() {
+ return offset;
+ }
+
+ public String getOffsetCursor() {
+ String cursor = "";
+ if(offset.isPresent()){
+ ByteBuffer buffer = INTEGER_SERIALIZER.toByteBuffer(offset.get());
+ cursor = Base64.encodeBase64String(buffer.array());
+ }
+ return cursor;
+ }
+
+ public void setOffsetFromCursor(String cursor) {
+ if(cursor == null || cursor.length() == 0){
+ clearOffset();
+ }else {
+ byte[] bytes = Base64.decodeBase64(cursor);
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
+ setOffset(number);
+ }
+ }
+
+ public void clearOffset() {
+ this.offset = Optional.absent();
+ }
+
+ public void setOffset( int offset ) {
+ this.offset = Optional.of(offset);
+ }
+
+
+ public Query withOffset( int offset ) {
+ setOffset(offset);
+ return this;
+ }
+
+ public Query withOffsetFromCursor( String cursor ) {
+ setOffsetFromCursor(cursor);
+ return this;
+ }
+
+
+ public int getLimit() {
+ return getLimit( DEFAULT_LIMIT );
+ }
+
+
+ public int getLimit( int defaultLimit ) {
+ if ( limit <= 0 ) {
+ if ( defaultLimit > 0 ) {
+ return defaultLimit;
+ }
+ else {
+ return DEFAULT_LIMIT;
+ }
+ }
+ return limit;
+ }
+
+
+ public void setLimit( int limit ) {
+
+ // TODO tnine. After users have had time to change their query limits,
+ // this needs to be uncommented and enforced.
+ // if(limit > MAX_LIMIT){
+ // throw new IllegalArgumentException(
+ // String.format("Query limit must be <= to %d", MAX_LIMIT));
+ // }
+
+ if ( limit > MAX_LIMIT ) {
+ limit = MAX_LIMIT;
+ }
+
+ this.limit = limit;
+ }
+
+
+ public Query withLimit( int limit ) {
+ setLimit( limit );
+ return this;
+ }
+
+
+ public boolean isReversed() {
+ return reversed;
+ }
+
+
+ public void setReversed( boolean reversed ) {
+ reversedSet = true;
+ this.reversed = reversed;
+ }
+
+
+ public boolean isReversedSet() {
+ return reversedSet;
+ }
+
+
+ public Long getStartTime() {
+ return startTime;
+ }
+
+
+ public void setStartTime( Long startTime ) {
+ this.startTime = startTime;
+ }
+
+
+ public Long getFinishTime() {
+ return finishTime;
+ }
+
+
+ public void setFinishTime( Long finishTime ) {
+ this.finishTime = finishTime;
+ }
+
+
+ public boolean isPad() {
+ return pad;
+ }
+
+
+ public void setPad( boolean pad ) {
+ this.pad = pad;
+ }
+
+
+ public void setResolution( CounterResolution resolution ) {
+ this.resolution = resolution;
+ }
+
+
+ public CounterResolution getResolution() {
+ return resolution;
+ }
+
+
+ public void addIdentifier( Identifier identifier ) {
+ if ( identifiers == null ) {
+ identifiers = new ArrayList<Identifier>();
+ }
+ identifiers.add( identifier );
+ }
+
+
+ void setIdentifiers( List<Identifier> identifiers ) {
+ this.identifiers = identifiers;
+ }
+
+
+ public List<CounterFilterPredicate> getCounterFilters() {
+ return counterFilters;
+ }
+
+
+ public void addCounterFilter( String counter ) {
+ CounterFilterPredicate p = CounterFilterPredicate.fromString( counter );
+ if ( p == null ) {
+ return;
+ }
+ if ( counterFilters == null ) {
+ counterFilters = new ArrayList<CounterFilterPredicate>();
+ }
+ counterFilters.add( p );
+ }
+
+
+ void setCounterFilters( List<CounterFilterPredicate> counterFilters ) {
+ this.counterFilters = counterFilters;
+ }
+
+
+
+ public static final class CounterFilterPredicate implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final String name;
+ private final Identifier user;
+ private final Identifier group;
+ private final String queue;
+ private final String category;
+
+
+ public CounterFilterPredicate( String name, Identifier user, Identifier group, String queue, String category ) {
+ this.name = name;
+ this.user = user;
+ this.group = group;
+ this.queue = queue;
+ this.category = category;
+ }
+
+
+ public Identifier getUser() {
+ return user;
+ }
+
+
+ public Identifier getGroup() {
+ return group;
+ }
+
+
+ public String getQueue() {
+ return queue;
+ }
+
+
+ public String getCategory() {
+ return category;
+ }
+
+
+ public String getName() {
+ return name;
+ }
+
+
+ public static CounterFilterPredicate fromString( String s ) {
+ Identifier user = null;
+ Identifier group = null;
+ String category = null;
+ String name = null;
+ String[] l = StringUtils.split( s, ':' );
+
+ if ( l.length > 0 ) {
+ if ( !"*".equals( l[0] ) ) {
+ name = l[0];
+ }
+ }
+
+ if ( l.length > 1 ) {
+ if ( !"*".equals( l[1] ) ) {
+ user = Identifier.from( l[1] );
+ }
+ }
+
+ if ( l.length > 2 ) {
+ if ( !"*".equals( l[2] ) ) {
+ group = Identifier.from( l[3] );
+ }
+ }
+
+ if ( l.length > 3 ) {
+ if ( !"*".equals( l[3] ) ) {
+ category = l[3];
+ }
+ }
+
+ if ( ( user == null ) && ( group == null ) && ( category == null ) && ( name == null)) {
+ return null;
+ }
+
+ return new CounterFilterPredicate( name, user, group, null, category );
+ }
+
+
+ public static List<CounterFilterPredicate> fromList( List<String> l ) {
+ if ( ( l == null ) || ( l.size() == 0 ) ) {
+ return null;
+ }
+ List<CounterFilterPredicate> counterFilters = new ArrayList<CounterFilterPredicate>();
+ for ( String s : l ) {
+ CounterFilterPredicate filter = CounterFilterPredicate.fromString( s );
+ if ( filter != null ) {
+ counterFilters.add( filter );
+ }
+ }
+ if ( counterFilters.size() == 0 ) {
+ return null;
+ }
+ return counterFilters;
+ }
+ }
+
+
+ // public List<Object> getSelectionResults( Results rs ) {
+ //
+ // List<Entity> entities = rs.getEntities();
+ // if ( entities == null ) {
+ // return null;
+ // }
+ //
+ // if ( !hasSelectSubjects() ) {
+ // return cast( entities );
+ // }
+ //
+ // List<Object> results = new ArrayList<Object>();
+ //
+ // for ( Entity entity : entities ) {
+ // if ( isMergeSelectResults() ) {
+ // boolean include = false;
+ // Map<String, Object> result = new LinkedHashMap<String, Object>();
+ // Map<String, String> selects = getSelectAssignments();
+ // for ( Map.Entry<String, String> select : selects.entrySet() ) {
+ // Object obj = JsonUtils.select( entity, select.getValue(), false );
+ // if ( obj != null ) {
+ // include = true;
+ // }
+ // result.put( select.getKey(), obj );
+ // }
+ // if ( include ) {
+ // results.add( result );
+ // }
+ // }
+ // else {
+ // boolean include = false;
+ // List<Object> result = new ArrayList<Object>();
+ // Set<String> selects = getSelectSubjects();
+ // for ( String select : selects ) {
+ // Object obj = JsonUtils.select( entity, select );
+ // if ( obj != null ) {
+ // include = true;
+ // }
+ // result.add( obj );
+ // }
+ // if ( include ) {
+ // results.add( result );
+ // }
+ // }
+ // }
+ //
+ // if ( results.size() == 0 ) {
+ // return null;
+ // }
+ //
+ // return results;
+ // }
+
+
+ // public Object getSelectionResult( Results rs ) {
+ // List<Object> r = getSelectionResults( rs );
+ // if ( ( r != null ) && ( r.size() > 0 ) ) {
+ // return r.get( 0 );
+ // }
+ // return null;
+ // }
+
+
+ private static String decode( String input ) {
+ try {
+ return URLDecoder.decode( input, "UTF-8" );
+ }
+ catch ( UnsupportedEncodingException e ) {
+ // shouldn't happen, but just in case
+ throw new RuntimeException( e );
+ }
+ }
+
+
+ // note: very likely to be null
+ public String getCollection() {
+ return collection;
+ }
+
+
+ public void setCollection( String collection ) {
+ this.collection = collection;
+ }
+
+
+ // may be null
+ public Optional<String> getQl() {
+
+ //if a query exists, but with no ql, we select all
+ return Optional.fromNullable( ql );
+ }
+
+
++ /**
++ * Return true if no query is present and we should perform a graph search
++ * @return
++ */
++ public boolean graphSearch(){
++ return ql == null;
++ }
++
+ public Query setQl( String ql ) {
+ this.ql = ql;
+ return this;
+ }
+
+
+ /**
+ * Get the connection type
+ * @return
+ */
+ public String getConnectionType() {
+ return connectionType;
+ }
+
+
+ /**
+ * Set the connection type
+ * @param connection
+ * @return
+ */
+ public Query setConnectionType( final String connection ) {
+ this.connectionType = connection;
+ return this;
+ }
+
+
+ public String getType() {
+ return type;
+ }
+
+
+ public Level getLevel() {
+ return level;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index 03f1df8,13187a7..10ee0f6
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@@ -28,11 -28,8 +28,9 @@@ import java.util.UUID
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.usergrid.persistence.core.guice.ProxyImpl;
- import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index fcab21f,eb59625..7f2ff09
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@@ -89,26 -76,19 +76,22 @@@ public interface IndexFig extends Guicy
/**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ * Timeout calls to elasticsearch.
+ * @return
*/
- public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
-
+ String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
- public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
/**
- * Timeout calls to elasticsearch.
- * @return
+ * The client type to use. Valid values are NODE or TRANSPORT
*/
- public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+ String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
- public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
+
++ String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
String getHosts();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/pom.xml
----------------------------------------------------------------------
[11/52] [abbrv] incubator-usergrid git commit: Merge remote-tracking
branch 'origin/two-dot-o-dev' into USERGRID-541
Posted by to...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o-dev' into USERGRID-541
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/48d8060e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/48d8060e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/48d8060e
Branch: refs/heads/USERGRID-593
Commit: 48d8060e8642e7fb5333f6d65b2ba90b1a5ec292
Parents: 745f355 8f592dd
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 15:31:25 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 15:31:25 2015 -0600
----------------------------------------------------------------------
portal/index-template.html | 2 +-
portal/js/data/data-controller.js | 58 +++++++++-
portal/js/data/data.html | 36 +++++-
portal/js/global/ug-service.js | 30 ++++-
portal/js/login/login-controller.js | 15 ++-
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../persistence/EntityManagerFactory.java | 6 +-
.../DuplicateUniquePropertyExistsException.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 2 +-
.../persistence/index/IndexRefreshCommand.java | 23 +++-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../index/impl/IndexRefreshCommandImpl.java | 24 ++--
.../src/test/resources/dynamic-test.properties | 2 +-
stack/pom.xml | 12 +-
stack/rest/pom.xml | 16 ++-
.../org/apache/usergrid/rest/IndexResource.java | 4 +-
.../org/apache/usergrid/rest/RootResource.java | 11 +-
.../rest/test/resource2point0/README.md | 111 +++++++++++++++++++
.../resource2point0/endpoints/RootResource.java | 1 +
.../security/providers/AbstractProvider.java | 6 +
.../providers/PingIdentityProvider.java | 14 ++-
.../services/AbstractCollectionService.java | 2 +-
23 files changed, 333 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 961ddd2,c80f80d..fcfba15
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -320,10 -346,10 +320,10 @@@ public class EsEntityIndexImpl implemen
- public Observable<Boolean> refreshAsync() {
+ public Observable<IndexRefreshCommand.IndexRefreshCommandInfo> refreshAsync() {
refreshIndexMeter.mark();
- BetterFuture future = indexBatchBufferProducer.put(new IndexIdentifierImpl.IndexOperationMessage());
+ BetterFuture future = producer.put(new IndexOperationMessage());
future.get();
return indexRefreshCommand.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48d8060e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
index 5052ddf,804de51..07150bb
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRefreshCommandImpl.java
@@@ -143,22 -145,24 +145,24 @@@ public class IndexRefreshCommandImpl im
return future.doOnNext( found -> {
- if ( !found ) {
- logger.error( "Couldn't find record during refresh uuid" + uuid );
+ if ( !found.hasFinished() ) {
+ logger.error(String.format("Couldn't find record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
+ }else{
+ logger.info(String.format("found record during refresh uuid: {} took ms:{} ", uuid, found.getExecutionTime()));
}
- } ).doOnCompleted( () -> {
+ } ).doOnCompleted(() -> {
//clean up our data
- String[] aliases = indexCache.getIndexes( alias, AliasedEntityIndex.AliasType.Read );
+ String[] aliases = indexCache.getIndexes(alias, AliasedEntityIndex.AliasType.Read);
DeIndexRequest deIndexRequest =
- new DeIndexRequest( aliases, appScope, edge, entity.getId(), entity.getVersion() );
+ new DeIndexRequest(aliases, appScope, edge, entity.getId(), entity.getVersion());
//delete the item
- IndexIdentifierImpl.IndexOperationMessage indexOperationMessage =
- new IndexIdentifierImpl.IndexOperationMessage();
- indexOperationMessage.addDeIndexRequest(deIndexRequest);
- producer.put(indexOperationMessage);
+ IndexOperationMessage indexOperationMessage =
+ new IndexOperationMessage();
+ indexOperationMessage.addDeIndexRequest( deIndexRequest );
+ producer.put( indexOperationMessage );
refreshTimer.stop();
- } );
+ });
}
}
[50/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/852c9747
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/852c9747
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/852c9747
Branch: refs/heads/USERGRID-593
Commit: 852c9747bb8c88401a343d971a68203ee19459c5
Parents: e91930f 0141623
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Apr 23 10:24:28 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Apr 23 10:24:28 2015 -0600
----------------------------------------------------------------------
.../Usergrid.Notifications/BackgroundTask.cs | 57 +++++++++++++++++++
.../Usergrid.Notifications/Client/PushClient.cs | 12 +++-
.../Usergrid.Notifications/MainPage.xaml.cs | 12 ++--
.../Usergrid.Notifications/MyBackgroundTask.cs | 18 ++++++
.../Usergrid.Notifications/Package.appxmanifest | 8 +++
.../Usergrid.Notifications.csproj | 1 +
.../Usergrid.Notifications.sln | 40 +++++++++++++
.../persistence/query/IteratingQueryIT.java | 54 +++++++++---------
.../services/notifications/wns/WNSAdapter.java | 60 +++++++-------------
9 files changed, 189 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
[47/52] [abbrv] incubator-usergrid git commit: add wns changes
Posted by to...@apache.org.
add wns changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/24e7cda7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/24e7cda7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/24e7cda7
Branch: refs/heads/USERGRID-593
Commit: 24e7cda7b15c028b00de3285466c8abd9840be87
Parents: 381d8c5
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Apr 23 09:31:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Apr 23 09:31:16 2015 -0600
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 2 +-
.../Usergrid.Notifications/MainPage.xaml.cs | 6 +-
.../services/notifications/wns/WNSAdapter.java | 60 +++++++-------------
4 files changed, 26 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24e7cda7/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
index 5279e32..5efbb9b 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
@@ -108,7 +108,7 @@ namespace Usergrid.Notifications.Client
/// <param name="message"></param>
/// <returns></returns>
Task<bool> SendToast(string message);
- Task<bool> SendRaw(JObject message);
+ Task<bool> SendRaw(string message);
/// <summary>
/// Send a badge update
/// </summary>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24e7cda7/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
index 54d7b00..7ccb391 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
@@ -86,7 +86,7 @@ namespace Usergrid.Notifications.Client
}
- public async Task<bool> SendRaw(JObject message)
+ public async Task<bool> SendRaw(string message)
{
if (DeviceId == null)
{
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24e7cda7/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index 80b6dd8..2625289 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -45,7 +45,7 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "http://yoururl/";
+ serverUrl = "http://10.0.1.20:8080/";
//TODO: change me to your org
org = "test-organization";
//TODO: change me to your app
@@ -55,7 +55,7 @@ namespace Usergrid.Notifications
//TODO: change me to your user
user = "mobileuser";
//TODO: change me to your password
- password = "password";
+ password = "P@ssw0rd1";
this.NavigationCacheMode = NavigationCacheMode.Required;
usergrid = new Client.Usergrid(serverUrl, org, app, user, password, notifier);
@@ -130,7 +130,7 @@ namespace Usergrid.Notifications
JObject obj = new JObject();
obj.Add("message", message);
- if (await usergrid.Push.SendRaw(obj))
+ if (await usergrid.Push.SendRaw(message))
{
Result.Text = "It did! :)";
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/24e7cda7/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
index 34ae63f..1fe8f72 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
@@ -21,10 +21,9 @@
package org.apache.usergrid.services.notifications.wns;
import ar.com.fernandospr.wns.WnsService;
-import ar.com.fernandospr.wns.model.WnsBadge;
-import ar.com.fernandospr.wns.model.WnsRaw;
-import ar.com.fernandospr.wns.model.WnsToast;
+import ar.com.fernandospr.wns.model.*;
import ar.com.fernandospr.wns.model.builders.WnsBadgeBuilder;
+import ar.com.fernandospr.wns.model.builders.WnsRawBuilder;
import ar.com.fernandospr.wns.model.builders.WnsToastBuilder;
import com.sun.jersey.api.client.ClientHandlerException;
import org.apache.usergrid.persistence.EntityManager;
@@ -64,9 +63,9 @@ public class WNSAdapter implements ProviderAdapter {
WnsToast toast = new WnsToastBuilder().bindingTemplateToastText01("test").build();
try{
//this fails every time due to jax error which is ok
- service.pushToast("s-1-15-2-2411381248-444863693-3819932088-4077691928-1194867744-112853457-373132695",toast);
+ service.pushToast("s-1-15-2-2411381248-444863693-3819932088-4077691928-1194867744-112853457-373132695", toast);
}catch (ClientHandlerException e){
- LOG.info("Windows Phone notifier added: "+e.toString());
+ LOG.info("Windows Phone notifier added: " + e.toString());
}
}
@@ -89,12 +88,23 @@ public class WNSAdapter implements ProviderAdapter {
}
service.pushBadge(providerId, badge);
break;
- case "raw" :
- WnsRaw raw = new WnsRaw();
- raw.stream = toBytes( translatedNotification.getMessage() ) ;
- service.pushRaw(providerId, raw);
+ case "raw":
+ Object message = translatedNotification.getMessage();
+ if(message instanceof String) {
+ WnsRaw raw = new WnsRawBuilder().stream(((String) message).getBytes()).build();
+ WnsNotificationRequestOptional opt = new WnsNotificationRequestOptional();
+ opt.cachePolicy = "cache";
+ opt.requestForStatus = "true";
+ WnsNotificationResponse response = service.pushRaw(providerId, opt, raw);
+ if (!response.notificationStatus.equals("received")) { // https://msdn.microsoft.com/en-us/library/windows/apps/hh465435.aspx#pncodes_x_wns_notification
+ throw new Exception(String.format("Notification failed status:%s, devicesStatus:%s, description:%s, debug flag:%s", response.notificationStatus, response.deviceConnectionStatus, response.errorDescription, response.debugTrace));
+ }
+ }else{
+ throw new IllegalArgumentException("You must send a string in the raw body. instead got this: " + message.getClass().getName());
+ }
break;
- default : throw new IllegalArgumentException(translatedNotification.getType()+" does not match a valid notification type (toast,badge).");
+ default:
+ throw new IllegalArgumentException(translatedNotification.getType() + " does not match a valid notification type (toast,badge).");
}
}
tracker.completed();
@@ -104,35 +114,7 @@ public class WNSAdapter implements ProviderAdapter {
}
}
- private byte[] toBytes(Object message) {
- ByteArrayOutputStream bos = new ByteArrayOutputStream();
- ObjectOutput out = null;
- try {
- if(message instanceof Serializable) {
- out = new ObjectOutputStream(bos);
- out.writeObject(message);
- byte[] yourBytes = bos.toByteArray();
- return yourBytes;
- }else{
- throw new RuntimeException("message is not serializable");
- }
- }catch (IOException e){
- throw new RuntimeException(e);
- } finally {
- try {
- if (out != null) {
- out.close();
- }
- } catch (IOException ex) {
- // ignore close exception
- }
- try {
- bos.close();
- } catch (IOException ex) {
- // ignore close exception
- }
- }
- }
+
@Override
public void doneSendingNotifications() throws Exception {
[38/52] [abbrv] incubator-usergrid git commit: null reference
Posted by to...@apache.org.
null reference
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/46a0e8f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/46a0e8f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/46a0e8f3
Branch: refs/heads/USERGRID-593
Commit: 46a0e8f32a65fca3ddb601d3991dac06c8040b99
Parents: a3cacb6
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 14:59:35 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 14:59:35 2015 -0600
----------------------------------------------------------------------
.../results/ElasticSearchQueryExecutor.java | 26 +++++++++-----------
.../org/apache/usergrid/persistence/Query.java | 3 +++
.../impl/EsApplicationEntityIndexImpl.java | 17 ++++++-------
3 files changed, 22 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/46a0e8f3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index ca16bc0..98a8dac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -23,16 +23,13 @@ package org.apache.usergrid.corepersistence.results;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import org.apache.usergrid.persistence.index.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
import com.google.common.base.Optional;
@@ -118,7 +115,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
// need to query for more
// ask for just what we need to satisfy, don't want to exceed limit
- query.setOffsetFromCursor( results.getCursor() );
+ query.setOffsetFromCursor(results.getCursor());
query.setLimit( originalLimit - results.size() );
logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
@@ -128,7 +125,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//now set our cursor if we have one for the next iteration
if ( results.hasCursor() ) {
- query.setOffsetFromCursor( results.getCursor() );
+ query.setOffsetFromCursor(results.getCursor());
moreToLoad = true;
}
@@ -157,15 +154,14 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
final Optional<Integer> cursor = query.getOffset();
final String queryToExecute = query.getQl().or("select *");
- if(cursor.isPresent()){
- //since query is a nasty stateful builder object, we have to default to select * if a query is issued
- //from legacy code with no QL set. An empty query is functionally equivalent to select all with default
- //sort ordering
-
- return entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get());
+ CandidateResults results = cursor.isPresent()
+ ? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
+ : entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
+ //set offset into query
+ if(results.getOffset().isPresent()) {
+ query.setOffset(results.getOffset().get());
}
-
- return entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
+ return results;
}
@@ -184,7 +180,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
//load the results
- final Results results = resultsLoader.loadResults( crs );
+ final Results results = resultsLoader.loadResults(crs);
//signal for post processing
resultsLoader.postProcess();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/46a0e8f3/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 9f0baf3..4273334 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -528,6 +528,9 @@ public class Query {
}
public void setOffsetFromCursor(String cursor) {
+ if(cursor == null || cursor.length() == 0){
+ return;
+ }
byte[] bytes = Base64.decodeBase64(cursor);
ByteBuffer buffer = ByteBuffer.wrap(bytes);
Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/46a0e8f3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 89d6da7..4b31bf0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -121,7 +121,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
this.esProvider = provider;
mapManager = mapManagerFactory.createMapManager( mapScope );
- this.searchTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.timer" );
+ this.searchTimer = metricsFactory.getTimer(EsApplicationEntityIndexImpl.class, "search.timer");
this.cursorTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.cursor.timer" );
this.cursorTimeout = config.getQueryCursorTimeout();
this.queryTimeout = config.getWriteTimeout();
@@ -147,13 +147,13 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
@Override
public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
final int limit ) {
- return search(searchEdge,searchTypes,query,limit,0);
+ return search(searchEdge, searchTypes, query, limit, 0);
}
public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
final int limit, final int offset ) {
- IndexValidationUtils.validateSearchEdge( searchEdge );
+ IndexValidationUtils.validateSearchEdge(searchEdge);
Preconditions.checkNotNull( searchTypes, "searchTypes cannot be null" );
Preconditions.checkNotNull( query, "query cannot be null" );
Preconditions.checkArgument( limit > 0, "limit must be > 0" );
@@ -184,7 +184,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
}
failureMonitor.success();
- return parseResults( searchResponse, parsedQuery, limit, offset );
+ return parseResults(searchResponse, parsedQuery, limit, offset);
}
@@ -247,11 +247,10 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final SearchHits searchHits = searchResponse.getHits();
final SearchHit[] hits = searchHits.getHits();
- final int length = hits.length;
- logger.debug( " Hit count: {} Total hits: {}", length, searchHits.getTotalHits() );
+ logger.debug( " Hit count: {} Total hits: {}", hits.length, searchHits.getTotalHits() );
- List<CandidateResult> candidates = new ArrayList<>( length );
+ List<CandidateResult> candidates = new ArrayList<>( hits.length );
for ( SearchHit hit : hits ) {
@@ -263,9 +262,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings());
// >= seems odd. However if we get an overflow, we need to account for it.
- if ( length >= limit ) {
+ if ( hits.length >= limit ) {
- candidateResults.initializeCursor(from);
+ candidateResults.initializeCursor(from+limit);
}
[09/52] [abbrv] incubator-usergrid git commit: merge two-o,
remove java-sdk old
Posted by to...@apache.org.
merge two-o, remove java-sdk old
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0db7f4c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0db7f4c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0db7f4c0
Branch: refs/heads/USERGRID-593
Commit: 0db7f4c09f38a3cb961cecd866580555175e1d4f
Parents: 8f592dd 527ac86
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 14:54:11 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 14:54:11 2015 -0600
----------------------------------------------------------------------
.gitattributes | 17 +
.gitignore | 4 +
.usergridversion | 1 +
CHANGELOG | 31 +
NOTICE | 17 +-
docs/.gitignore | 2 +
docs/Makefile | 177 +
docs/README | 31 +
docs/_theme/sphinx_rtd_theme/__init__.py | 17 +
docs/_theme/sphinx_rtd_theme/breadcrumbs.html | 19 +
docs/_theme/sphinx_rtd_theme/footer.html | 33 +
docs/_theme/sphinx_rtd_theme/layout.html | 162 +
docs/_theme/sphinx_rtd_theme/layout_old.html | 205 +
docs/_theme/sphinx_rtd_theme/search.html | 50 +
docs/_theme/sphinx_rtd_theme/searchbox.html | 7 +
.../sphinx_rtd_theme/static/css/badge_only.css | 2 +
.../sphinx_rtd_theme/static/css/theme.css | 5 +
.../static/fonts/FontAwesome.otf | Bin 0 -> 62856 bytes
.../static/fonts/fontawesome-webfont.eot | Bin 0 -> 38205 bytes
.../static/fonts/fontawesome-webfont.svg | 414 +
.../static/fonts/fontawesome-webfont.ttf | Bin 0 -> 80652 bytes
.../static/fonts/fontawesome-webfont.woff | Bin 0 -> 44432 bytes
docs/_theme/sphinx_rtd_theme/static/js/theme.js | 47 +
docs/_theme/sphinx_rtd_theme/theme.conf | 8 +
docs/_theme/sphinx_rtd_theme/versions.html | 37 +
docs/concepts/activity.md | 1121 ++
docs/concepts/applications.md | 78 +
docs/concepts/assets.md | 20 +
docs/concepts/collections.md | 21 +
docs/concepts/events-and-counters.md | 268 +
docs/concepts/groups.md | 38 +
docs/concepts/query-language.md | 427 +
docs/concepts/relationships.md | 100 +
docs/concepts/roles-and-permissions.md | 18 +
docs/concepts/users-devices.md | 108 +
docs/conf.py | 273 +
docs/contribute-code.md | 24 +
docs/deploy-local.md | 179 +
docs/getting-up-and-running-locally.md | 72 +
docs/index.md | 12 +
docs/lib/sphinxcontrib_markdown.py | 58 +
docs/make.bat | 242 +
docs/organizations-admins.md | 55 +
docs/presentations-and-videos.md | 7 +
docs/presentations-and-videos/presentations.md | 21 +
docs/presentations-and-videos/videos.md | 11 +
docs/upgrading.md | 3 +
docs/usage.md | 16 +
portal/Gruntfile.js | 20 +-
portal/README.md | 35 +-
portal/build.sh | 1 +
portal/config.js | 3 +-
portal/css/main.css | 23 +-
portal/img/sdk-sprites-large.png | Bin 14642 -> 15115 bytes
portal/img/sdk-sprites.png | Bin 5027 -> 4401 bytes
portal/index-template.html | 1 +
portal/js/global/help-service.js | 2 +-
portal/js/global/ug-service.js | 5 +-
portal/package.json | 2 +-
portal/pom.xml | 98 +
release/README.txt | 43 +
release/changelog.rb | 150 +
release/pom.xml | 65 -
release/release-candidate.sh | 296 +
release/release.sh | 248 +
release/sign-release.sh | 30 -
release/src/main/assembly/source.xml | 57 -
sdks/android/.gitignore | 20 +
sdks/android/NOTICE | 12 +
sdks/android/README.md | 33 +
sdks/android/README.txt | 2 -
sdks/android/assembly.xml | 55 +
sdks/android/build_release_zip.sh | 10 +
sdks/android/pom.xml | 76 +-
.../apache/usergrid/android/client/Client.java | 1036 --
.../client/callbacks/ApiResponseCallback.java | 26 -
.../client/callbacks/ClientAsyncTask.java | 54 -
.../client/callbacks/ClientCallback.java | 25 -
.../callbacks/DeviceRegistrationCallback.java | 25 -
.../callbacks/GroupsRetrievedCallback.java | 29 -
.../client/callbacks/QueryResultsCallback.java | 25 -
.../android/client/utils/DeviceUuidFactory.java | 170 -
.../android/client/utils/ObjectUtils.java | 36 -
.../usergrid/android/sdk/CounterIncrement.java | 72 +
.../sdk/DefaultURLConnectionFactory.java | 36 +
.../apache/usergrid/android/sdk/UGClient.java | 3179 +++++
.../android/sdk/URLConnectionFactory.java | 30 +
.../sdk/callbacks/ApiResponseCallback.java | 31 +
.../android/sdk/callbacks/ClientAsyncTask.java | 66 +
.../android/sdk/callbacks/ClientCallback.java | 31 +
.../sdk/callbacks/GroupsRetrievedCallback.java | 35 +
.../sdk/callbacks/QueryResultsCallback.java | 33 +
.../usergrid/android/sdk/entities/Activity.java | 1019 ++
.../android/sdk/entities/Collection.java | 338 +
.../usergrid/android/sdk/entities/Device.java | 122 +
.../usergrid/android/sdk/entities/Entity.java | 552 +
.../usergrid/android/sdk/entities/Group.java | 151 +
.../usergrid/android/sdk/entities/Message.java | 159 +
.../usergrid/android/sdk/entities/User.java | 315 +
.../android/sdk/exception/ClientException.java | 43 +
.../android/sdk/response/AggregateCounter.java | 58 +
.../sdk/response/AggregateCounterSet.java | 117 +
.../android/sdk/response/ApiResponse.java | 774 ++
.../sdk/response/ClientCredentialsInfo.java | 64 +
.../android/sdk/response/QueueInfo.java | 47 +
.../android/sdk/utils/DeviceUuidFactory.java | 173 +
.../usergrid/android/sdk/utils/JsonUtils.java | 185 +
.../usergrid/android/sdk/utils/MapUtils.java | 42 +
.../usergrid/android/sdk/utils/ObjectUtils.java | 39 +
.../usergrid/android/sdk/utils/UrlUtils.java | 127 +
sdks/dotnet/NOTICE | 56 +
.../NSubstitute.1.6.0.0.nupkg | Bin 619760 -> 0 bytes
.../NSubstitute.1.6.0.0.nuspec | 16 -
.../packages/NSubstitute.1.6.0.0/README.txt | 0
.../NSubstitute.1.6.0.0/acknowledgements.txt | 14 -
.../lib/NET35/NSubstitute.XML | 432 -
.../lib/NET35/NSubstitute.dll | Bin 1039360 -> 0 bytes
.../lib/NET40/NSubstitute.XML | 432 -
.../lib/NET40/NSubstitute.dll | Bin 561152 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nupkg | Bin 94702 -> 0 bytes
.../packages/NUnit.2.6.2/NUnit.2.6.2.nuspec | 16 -
sdks/dotnet/packages/NUnit.2.6.2/README.txt | 3 -
.../NUnit.2.6.2/lib/nunit.framework.dll | Bin 147456 -> 0 bytes
.../NUnit.2.6.2/lib/nunit.framework.xml | 10915 -----------------
.../packages/Newtonsoft.Json.4.5.11/LICENSE.txt | 21 +
.../Newtonsoft.Json.4.5.11.nupkg | Bin 1409718 -> 0 bytes
.../Newtonsoft.Json.4.5.11.nuspec | 16 -
.../packages/Newtonsoft.Json.4.5.11/README.txt | 3 -
.../lib/net20/Newtonsoft.Json.dll | Bin 406016 -> 0 bytes
.../lib/net20/Newtonsoft.Json.xml | 8542 -------------
.../lib/net35/Newtonsoft.Json.dll | Bin 366592 -> 0 bytes
.../lib/net35/Newtonsoft.Json.xml | 7678 ------------
.../lib/net40/Newtonsoft.Json.dll | Bin 391680 -> 0 bytes
.../lib/net40/Newtonsoft.Json.xml | 7921 ------------
.../Newtonsoft.Json.dll | Bin 315904 -> 0 bytes
.../Newtonsoft.Json.xml | 7107 -----------
.../lib/sl3-wp/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl3-wp/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4-windowsphone71/Newtonsoft.Json.dll | Bin 334336 -> 0 bytes
.../lib/sl4-windowsphone71/Newtonsoft.Json.xml | 7228 -----------
.../lib/sl4/Newtonsoft.Json.dll | Bin 337408 -> 0 bytes
.../lib/sl4/Newtonsoft.Json.xml | 7250 -----------
.../lib/winrt45/Newtonsoft.Json.dll | Bin 367616 -> 0 bytes
.../lib/winrt45/Newtonsoft.Json.xml | 7446 -----------
.../dotnet/packages/RestSharp.104.1/LICENSE.txt | 202 +
.../RestSharp.104.1/RestSharp.104.1.nupkg | Bin 605731 -> 0 bytes
.../RestSharp.104.1/RestSharp.104.1.nuspec | 16 -
.../lib/net35-client/RestSharp.dll | Bin 146944 -> 0 bytes
.../lib/net35-client/RestSharp.xml | 2627 ----
.../RestSharp.104.1/lib/net35/RestSharp.dll | Bin 146944 -> 0 bytes
.../RestSharp.104.1/lib/net35/RestSharp.xml | 2627 ----
.../lib/net4-client/RestSharp.dll | Bin 150016 -> 0 bytes
.../lib/net4-client/RestSharp.xml | 2696 ----
.../RestSharp.104.1/lib/net4/RestSharp.dll | Bin 150016 -> 0 bytes
.../RestSharp.104.1/lib/net4/RestSharp.xml | 2696 ----
.../lib/sl4-wp71/RestSharp.WindowsPhone.dll | Bin 165376 -> 0 bytes
.../lib/sl4-wp71/RestSharp.WindowsPhone.xml | 3510 ------
.../lib/sl4/RestSharp.Silverlight.dll | Bin 124928 -> 0 bytes
.../lib/sl4/RestSharp.Silverlight.xml | 2445 ----
sdks/html5-javascript/lib/modules/Client.js | 29 +-
sdks/nodejs/lib/usergrid.js | 31 +-
sdks/php/lib/vendor/Apache/Usergrid/Client.php | 33 +-
sdks/php5/apache-usergrid/.gitignore | 12 +
.../Examples/attrubites/attributes.php | 122 +
.../Examples/collections/books.php | 130 +
.../Examples/collections/data.php | 38 +
.../Examples/collections/users.php | 120 +
sdks/php5/apache-usergrid/Examples/examples.md | 17 +
sdks/php5/apache-usergrid/Examples/examples.php | 245 +
.../Examples/management/management.php | 91 +
.../apache-usergrid/Examples/messages/data.php | 17 +
.../Examples/messages/messages.php | 62 +
.../Examples/notifications/data.php | 17 +
.../Examples/notifications/notifications.php | 86 +
.../Examples/notifications/notifiers.php | 79 +
sdks/php5/apache-usergrid/LICENSE | 202 +
sdks/php5/apache-usergrid/README.md | 219 +
sdks/php5/apache-usergrid/TODO.md | 2 +
sdks/php5/apache-usergrid/composer.json | 40 +
sdks/php5/apache-usergrid/composer.lock | 1225 ++
sdks/php5/apache-usergrid/phpunit.xml | 29 +
.../src/Api/Exception/BadRequestException.php | 34 +
.../src/Api/Exception/InvalidIdException.php | 33 +
.../src/Api/Exception/NotFoundException.php | 33 +
.../src/Api/Exception/ServerErrorException.php | 33 +
.../src/Api/Exception/UnauthorizedException.php | 34 +
.../src/Api/Exception/UsergridException.php | 152 +
.../apache-usergrid/src/Api/Filters/Boolean.php | 44 +
.../apache-usergrid/src/Api/Filters/Date.php | 44 +
.../apache-usergrid/src/Api/Filters/Number.php | 48 +
.../apache-usergrid/src/Api/GuzzleClient.php | 64 +
.../src/Api/Models/Application.php | 35 +
.../src/Api/Models/BaseCollection.php | 116 +
.../src/Api/Models/Collection.php | 37 +
.../apache-usergrid/src/Api/Models/Device.php | 50 +
.../apache-usergrid/src/Api/Models/Entity.php | 36 +
.../apache-usergrid/src/Api/Models/Event.php | 35 +
.../apache-usergrid/src/Api/Models/Group.php | 54 +
.../src/Api/Models/GuzzleCommandTrait.php | 59 +
.../apache-usergrid/src/Api/Models/Notifier.php | 36 +
.../src/Api/Models/Organization.php | 38 +
.../apache-usergrid/src/Api/Models/Receipt.php | 36 +
.../apache-usergrid/src/Api/Models/Role.php | 54 +
.../apache-usergrid/src/Api/Models/User.php | 77 +
.../apache-usergrid/src/Api/QueryAggregator.php | 61 +
.../src/Api/ResourceIterator.php | 69 +
sdks/php5/apache-usergrid/src/Api/Usergrid.php | 452 +
.../Oauth2/GrantType/AuthorizationCode.php | 77 +
.../Oauth2/GrantType/ClientCredentials.php | 73 +
.../Oauth2/GrantType/GrantTypeInterface.php | 41 +
.../GrantType/HWIOAuthBundleRefreshToken.php | 62 +
.../Oauth2/GrantType/PasswordCredentials.php | 75 +
.../Plugin/Oauth2/GrantType/RefreshToken.php | 74 +
.../src/Guzzle/Plugin/Oauth2/Oauth2Plugin.php | 212 +
.../Laravel/ApacheUsergridServiceProvider.php | 159 +
.../src/Laravel/Facades/Usergrid.php | 42 +
.../src/Manifests/1.0.0/Application.php | 979 ++
.../src/Manifests/1.0.0/Errors.php | 53 +
.../src/Manifests/1.0.0/Management.php | 1076 ++
.../src/Manifests/1.0.0/Manifest.php | 25 +
.../src/Manifests/1.0.0/Notification.php | 154 +
.../src/Manifests/1.0.1/Activities.php | 24 +
.../src/Manifests/1.0.1/Application.php | 1108 ++
.../src/Manifests/1.0.1/Assets.php | 24 +
.../src/Manifests/1.0.1/Custom.php | 381 +
.../src/Manifests/1.0.1/Devices.php | 465 +
.../src/Manifests/1.0.1/Errors.php | 50 +
.../src/Manifests/1.0.1/Events.php | 24 +
.../src/Manifests/1.0.1/Groups.php | 486 +
.../src/Manifests/1.0.1/Management.php | 1076 ++
.../src/Manifests/1.0.1/Manifest.php | 24 +
.../src/Manifests/1.0.1/Messages.php | 19 +
.../src/Manifests/1.0.1/Notifications.php | 241 +
.../src/Manifests/1.0.1/Notifiers.php | 166 +
.../src/Manifests/1.0.1/Receipts.php | 24 +
.../src/Manifests/1.0.1/Roles.php | 381 +
.../src/Manifests/1.0.1/Users.php | 381 +
.../Manifests/swagger-json/applications.json | 1262 ++
.../src/Manifests/swagger-json/management.json | 1905 +++
.../src/Manifests/swagger-json/resources.json | 15 +
.../src/Native/ConfigRepository.php | 102 +
.../src/Native/Facades/Usergrid.php | 118 +
.../src/Native/UsergridBootstrapper.php | 150 +
sdks/php5/apache-usergrid/src/config/config.php | 53 +
.../tests/Api/ApplicationTest.php | 72 +
.../apache-usergrid/tests/Api/AttributeTest.php | 59 +
.../Api/Exception/BadRequestExceptionTest.php | 59 +
.../Api/Exception/InvalidIdExceptionTest.php | 58 +
.../Api/Exception/NotFoundExceptionTest.php | 57 +
.../Api/Exception/ServerErrorExceptionTest.php | 58 +
.../Api/Exception/UnauthorizedExceptionTest.php | 58 +
.../tests/Api/Filters/BooleanTest.php | 45 +
.../tests/Api/Filters/DateTest.php | 41 +
.../tests/Api/ManagementTest.php | 59 +
.../tests/Api/QueryAggregatorTest.php | 40 +
.../apache-usergrid/tests/Api/UsergridTest.php | 153 +
.../tests/Laravel/Facades/UsergridTest.php | 53 +
sdks/php5/apache-usergrid/tests/bootstrap.php | 61 +
stack/build-tools/pom.xml | 5 +-
stack/config/pom.xml | 1 +
.../main/resources/usergrid-default.properties | 37 +-
stack/core/pom.xml | 1 +
.../apache/usergrid/batch/job/package-info.java | 17 -
.../usergrid/batch/repository/package-info.java | 17 -
.../usergrid/batch/service/package-info.java | 17 -
.../org/apache/usergrid/count/package-info.java | 18 -
.../java/org/apache/usergrid/utils/BCrypt.java | 627 -
.../org/apache/usergrid/utils/package-info.java | 17 -
.../main/resources/usergrid-core-context.xml | 41 +-
.../apache/usergrid/batch/job/package-info.java | 17 -
.../query/OrderByDiscardPagesPagingIT.java | 164 +
.../org/apache/usergrid/utils/package-info.java | 17 -
.../collection/util/EntityHelper.java | 2 +-
stack/launcher/pom.xml | 8 +
.../java/org/apache/usergrid/launcher/App.java | 7 +-
.../launcher/ClasspathStaticHttpHandler.java | 321 -
.../org/apache/usergrid/launcher/Server.java | 5 +-
.../apache/usergrid/launcher/package-info.java | 17 -
stack/mongo-emulator/pom.xml | 3 +-
stack/pom.xml | 10 +-
stack/rest/pom.xml | 1 -
.../rest/applications/ServiceResource.java | 13 +-
.../rest/management/users/UsersResource.java | 14 +-
.../applications/assets/AssetResourceIT.java | 106 +
stack/services/pom.xml | 6 +
.../management/AccountCreationProps.java | 11 +
.../usergrid/management/ManagementService.java | 2 +
.../cassandra/AccountCreationPropsImpl.java | 8 +
.../cassandra/ManagementServiceImpl.java | 30 +-
.../security/crypto/command/BcryptCommand.java | 4 +-
.../security/providers/AbstractProvider.java | 5 +-
.../providers/PingIdentityProvider.java | 19 +-
stack/tools/pom.xml | 1 +
.../org/apache/usergrid/tools/ExportAdmins.java | 236 +
.../org/apache/usergrid/tools/ImportAdmins.java | 451 +
.../apache/usergrid/tools/ResetSuperUser.java | 18 +-
stack/websocket/pom.xml | 1 +
297 files changed, 31810 insertions(+), 91701 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/portal/index-template.html
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/portal/js/global/ug-service.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --cc stack/core/src/main/resources/usergrid-core-context.xml
index 0be6d00,4424896..37a2530
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@@ -158,7 -161,12 +161,9 @@@
</bean>
- <bean id="indexBucketLocator" class="org.apache.usergrid.persistence.cassandra.SimpleIndexBucketLocatorImpl">
- <constructor-arg value="${usergrid.index.defaultbucketsize}"/>
- </bean>
+ <bean id="entityManager" class="org.apache.usergrid.persistence.cassandra.EntityManagerImpl" scope="prototype"/>
+
<bean id="mailUtils" class="org.apache.usergrid.utils.MailUtils" />
<bean id="traceTagAspect" class="org.apache.usergrid.persistence.cassandra.util.TraceTagAspect"/>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/rest/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/services/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0db7f4c0/stack/services/src/main/java/org/apache/usergrid/security/providers/AbstractProvider.java
----------------------------------------------------------------------
[29/52] [abbrv] incubator-usergrid git commit: compiling in core and
services
Posted by to...@apache.org.
compiling in core and services
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/02defc1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/02defc1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/02defc1d
Branch: refs/heads/USERGRID-593
Commit: 02defc1d694b1b38e1871bb3a94f43ffb7c08691
Parents: 0830000
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 11:58:09 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 11:58:09 2015 -0600
----------------------------------------------------------------------
.../results/ElasticSearchQueryExecutor.java | 17 +++---
.../org/apache/usergrid/persistence/Query.java | 64 +++++++++++++-------
.../corepersistence/index/PublishRxtest.java | 2 +-
.../usergrid/persistence/CollectionIT.java | 12 ++--
.../org/apache/usergrid/persistence/GeoIT.java | 8 +--
.../apache/usergrid/persistence/IndexIT.java | 8 +--
.../PerformanceEntityRebuildIndexTest.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 30 ++++-----
.../persistence/query/NotSubPropertyIT.java | 2 +-
.../management/importer/ImportServiceImpl.java | 6 +-
12 files changed, 86 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index a9951bf..1a47717 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -107,7 +107,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
* This will only occur once during the repair phase.
* We need to ensure that we short circuit before we overflow the requested limit during a repair.
*/
- if ( crs.isEmpty() || !crs.hasCursor() || results.size() > 0 ) { // no results, no cursor, can't get more
+ if ( crs.isEmpty() || !crs.hasOffset() || results.size() > 0 ) { // no results, no cursor, can't get more
break;
}
@@ -118,7 +118,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
// need to query for more
// ask for just what we need to satisfy, don't want to exceed limit
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
query.setLimit( originalLimit - results.size() );
logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
@@ -128,7 +128,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//now set our cursor if we have one for the next iteration
if ( results.hasCursor() ) {
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
moreToLoad = true;
}
@@ -154,18 +154,18 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
* @return
*/
private CandidateResults getCandidateResults(final Query query){
- final String cursor = query.getCursor();
+ final Optional<Integer> cursor = query.getOffset();
+ final String queryToExecute = query.getQl().or("select *");
- if(cursor == null){
+ if(cursor.isPresent()){
//since query is a nasty stateful builder object, we have to default to select * if a query is issued
//from legacy code with no QL set. An empty query is functionally equivalent to select all with default
//sort ordering
- final String queryToExecute = query.getQl().or( "select *" );
return entityIndex.search( indexScope, types, queryToExecute, query.getLimit() );
}
- return entityIndex.getNextPage( cursor );
+ return entityIndex.search( indexScope, types, queryToExecute, query.getLimit(),cursor.get() );
}
@@ -189,8 +189,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//signal for post processing
resultsLoader.postProcess();
-
- results.setCursor( crs.getCursor() );
+ results.setCursor( query.getOffsetCursor() );
//ugly and tight coupling, but we don't have a choice until we finish some refactoring
results.setQueryExecutor( this );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 1f460d8..02a80af 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -30,6 +31,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;
+import com.netflix.astyanax.serializers.IntegerSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +55,7 @@ import com.google.common.base.Optional;
public class Query {
private static final Logger logger = LoggerFactory.getLogger( Query.class );
-
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
@@ -70,7 +72,7 @@ public class Query {
private String type;
private Operand rootOperand;
private UUID startResult;
- private String cursor;
+ private Optional<Integer> offset;
private int limit = 0;
private boolean mergeSelectResults = false;
@@ -111,7 +113,7 @@ public class Query {
ql = q.ql;
type = q.type;
startResult = q.startResult;
- cursor = q.cursor;
+ offset = q.offset;
limit = q.limit;
mergeSelectResults = q.mergeSelectResults;
//level = q.level;
@@ -206,12 +208,13 @@ public class Query {
List<Identifier> identifiers = null;
List<CounterFilterPredicate> counterFilters = null;
- String ql = QueryUtils.queryStrFrom( params );
- String type = ListUtils.first( params.get( "type" ) );
- Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
- String connection = ListUtils.first( params.get( "connectionType" ) );
- UUID start = ListUtils.firstUuid( params.get( "start" ) );
- String cursor = ListUtils.first( params.get( "cursor" ) );
+ String ql = QueryUtils.queryStrFrom(params);
+ String type = ListUtils.first(params.get("type"));
+ Boolean reversed = ListUtils.firstBoolean(params.get("reversed"));
+ String connection = ListUtils.first(params.get("connectionType"));
+ UUID start = ListUtils.firstUuid(params.get("start"));
+ String cursor = ListUtils.first(params.get("cursor"));
+
Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
@@ -267,9 +270,9 @@ public class Query {
q.setStartResult( start );
}
- if ( cursor != null ) {
+ if ( cursor != null) {
q = newQueryIfNull( q );
- q.setCursor( cursor );
+ q.setOffsetFromCursor(cursor);
}
if ( limit != null ) {
@@ -325,7 +328,7 @@ public class Query {
public static Query fromIdentifier( Object id ) {
Query q = new Query();
- q.addIdentifier( Identifier.from( id ) );
+ q.addIdentifier( Identifier.from(id) );
return q;
}
@@ -504,28 +507,43 @@ public class Query {
public UUID getStartResult() {
- if ( ( startResult == null ) && ( cursor != null ) ) {
- byte[] cursorBytes = Base64.decodeBase64( cursor );
- if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
- startResult = ConversionUtils.uuid( cursorBytes );
- }
- }
return startResult;
}
- public String getCursor() {
+ public Optional<Integer> getOffset() {
+ return offset;
+ }
+
+ public String getOffsetCursor() {
+ String cursor = "";
+ if(offset.isPresent()){
+ ByteBuffer buffer = INTEGER_SERIALIZER.toByteBuffer(offset.get());
+ cursor = Base64.encodeBase64String(buffer.array());
+ }
return cursor;
}
+ public void setOffsetFromCursor(String cursor) {
+ byte[] bytes = Base64.decodeBase64(cursor);
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
+ setOffset(number);
+ }
+
- public void setCursor( String cursor ) {
- this.cursor = cursor;
+ public void setOffset( int offset ) {
+ this.offset = Optional.of(offset);
}
- public Query withCursor( String cursor ) {
- setCursor( cursor );
+ public Query withOffset( int offset ) {
+ setOffset(offset);
+ return this;
+ }
+
+ public Query withOffsetFromCursor( String cursor ) {
+ setOffsetFromCursor(cursor);
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
index 1eb00c2..180aa86 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
@@ -36,7 +36,7 @@ import static org.junit.Assert.assertTrue;
/**
* Validates the Rx scheduler works as expected with publish
*/
-public class PublishRxTest {
+public class PublishRxtest {
@Test
public void testPublish() throws InterruptedException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index d1f746a..571c001 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -836,14 +836,14 @@ public class CollectionIT extends AbstractCoreIT {
query = Query.fromQL( "select * where NOT subObjectArray.subField = 'Bar'" ).withLimit( 1 )
- .withCursor( r.getCursor() );
+ .withOffsetFromCursor( r.getCursor() );
r = em.searchCollection( em.getApplicationRef(), "games", query );
assertEquals( 1, r.size() );
assertNotNull( r.getCursor() );
assertEquals( entity1, r.getEntities().get( 0 ) );
query = Query.fromQL( "select * where NOT subObjectArray.subField = 'Bar'" ).withLimit( 1 )
- .withCursor( r.getCursor() );
+ .withOffsetFromCursor( r.getCursor() );
r = em.searchCollection( em.getApplicationRef(), "games", query );
assertEquals( 0, r.size() );
assertNull( r.getCursor() );
@@ -984,7 +984,7 @@ public class CollectionIT extends AbstractCoreIT {
// try the next page, set our cursor, it should be the last 5 entities
query = new Query();
- query.setCursor( r.getCursor() );
+ query.setOffsetFromCursor( r.getCursor() );
r = em.searchCollection( em.getApplicationRef(), "objects", query );
@@ -1037,7 +1037,7 @@ public class CollectionIT extends AbstractCoreIT {
assertEquals( entityIds.get( i * pageSize + j ), r.getEntities().get( j ).getUuid() );
}
- query.setCursor( r.getCursor() );
+ query.setOffsetFromCursor( r.getCursor() );
}
//check our last search
@@ -1095,7 +1095,7 @@ public class CollectionIT extends AbstractCoreIT {
assertEquals( entityId, r.getEntities().get( j ).getUuid() );
}
- query.setCursor( r.getCursor() );
+ query.setOffsetFromCursor( r.getCursor() );
}
r = em.searchCollection( em.getApplicationRef(), "pages", query );
@@ -1148,7 +1148,7 @@ public class CollectionIT extends AbstractCoreIT {
assertEquals( expectedId, r.getEntities().get( j ).getUuid() );
}
- query.setCursor( r.getCursor() );
+ query.setOffsetFromCursor( r.getCursor() );
}
r = em.searchCollection( em.getApplicationRef(), "pages", query );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 614e5f6..1e62af2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -416,7 +416,7 @@ public class GeoIT extends AbstractCoreIT {
}
// set for the next "page"
- query.setCursor(results.getCursor());
+ query.setOffsetFromCursor(results.getCursor());
}
while (results.getCursor() != null);
@@ -462,7 +462,7 @@ public class GeoIT extends AbstractCoreIT {
}
// set for the next "page"
- query.setCursor(results.getCursor());
+ query.setOffsetFromCursor(results.getCursor());
}
while (results.getCursor() != null);
@@ -521,7 +521,7 @@ public class GeoIT extends AbstractCoreIT {
count++;
}
}
- while (query.getCursor() != null);
+ while (query.getOffset().isPresent());
// check we got back all 500 entities
assertEquals(numEntities, count);
@@ -577,7 +577,7 @@ public class GeoIT extends AbstractCoreIT {
count++;
}
- query.setCursor(r.getCursor());
+ query.setOffsetFromCursor(r.getCursor());
}
while (r.hasCursor());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index 97f0045..63bbbeb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -81,14 +81,14 @@ public class IndexIT extends AbstractCoreIT {
i++;
}
- query = Query.fromQL( "order by name" ).withCursor( r.getCursor() );
+ query = Query.fromQL( "order by name" ).withOffsetFromCursor(r.getCursor());
r = em.searchCollection( em.getApplicationRef(), "items", query );
for ( Entity entity : r.getEntities() ) {
assertEquals( alphabet[i], entity.getProperty( "name" ) );
i++;
}
- query = Query.fromQL( "order by name" ).withCursor( r.getCursor() );
+ query = Query.fromQL( "order by name" ).withOffsetFromCursor(r.getCursor());
r = em.searchCollection( em.getApplicationRef(), "items", query );
for ( Entity entity : r.getEntities() ) {
assertEquals( alphabet[i], entity.getProperty( "name" ) );
@@ -106,7 +106,7 @@ public class IndexIT extends AbstractCoreIT {
assertEquals( alphabet[i], entity.getProperty( "name" ) );
}
- query = Query.fromQL( "order by name desc" ).withCursor( r.getCursor() );
+ query = Query.fromQL( "order by name desc" ).withOffsetFromCursor(r.getCursor());
r = em.searchCollection( em.getApplicationRef(), "items", query );
// LOG.info(JsonUtils.mapToFormattedJsonString(r.getEntities()));
for ( Entity entity : r.getEntities() ) {
@@ -114,7 +114,7 @@ public class IndexIT extends AbstractCoreIT {
assertEquals( alphabet[i], entity.getProperty( "name" ) );
}
- query = Query.fromQL( "order by name desc" ).withCursor( r.getCursor() );
+ query = Query.fromQL( "order by name desc" ).withOffsetFromCursor( r.getCursor() );
r = em.searchCollection( em.getApplicationRef(), "items", query );
for ( Entity entity : r.getEntities() ) {
i--;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index a293a0c..df0b4ec 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -423,7 +423,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
if ( results.hasCursor() ) {
logger.info( "Counted {} : query again with cursor", count);
- q.setCursor( results.getCursor() );
+ q.setOffsetFromCursor( results.getCursor() );
results = em.searchCollection( em.getApplicationRef(), collectionName, q );
} else {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java
index 3b12353..e2fb02e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionTransitivePagingIT.java
@@ -163,7 +163,7 @@ public class IntersectionTransitivePagingIT{
returnedUUID );
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java
index e8ee0b6..fc004e8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IntersectionUnionPagingIT.java
@@ -169,7 +169,7 @@ public class IntersectionUnionPagingIT {
newSets.remove( name );
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
index 6466823..3a9c5f2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/IteratingQueryIT.java
@@ -321,7 +321,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -392,7 +392,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -462,7 +462,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -575,7 +575,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -645,7 +645,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -764,7 +764,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -824,7 +824,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -883,7 +883,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -942,7 +942,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -1005,7 +1005,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -1068,7 +1068,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.hasCursor() );
@@ -1127,7 +1127,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -1233,7 +1233,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -1337,7 +1337,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
@@ -1401,7 +1401,7 @@ public class IteratingQueryIT {
count++;
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java
index ff66f8e..11e0f09 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/NotSubPropertyIT.java
@@ -163,7 +163,7 @@ public class NotSubPropertyIT {
assertEquals( "Not returned as excpected", expected, returned );
}
- query.setCursor( results.getCursor() );
+ query.setOffsetFromCursor( results.getCursor() );
}
while ( results.getCursor() != null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/02defc1d/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 18879c0..c86bfc7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -144,7 +144,7 @@ public class ImportServiceImpl implements ImportService {
final Entity appInfo = getApplicationInfoEntity(rootEm, applicationId);
Query query = Query.fromQLNullSafe( ql );
- query.setCursor( cursor );
+ query.setOffsetFromCursor( cursor );
//set our entity type
query.setEntityType( Schema.getDefaultSchema().getEntityType( Import.class ) );
@@ -205,7 +205,7 @@ public class ImportServiceImpl implements ImportService {
final Import importEntity = getImport( applicationId, importId );
Query query = Query.fromQLNullSafe( ql );
- query.setCursor( cursor );
+ query.setOffsetFromCursor( cursor );
query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
query.setResultsLevel( Level.ALL_PROPERTIES );
@@ -264,7 +264,7 @@ public class ImportServiceImpl implements ImportService {
final FileImport importEntity = getFileImport(applicationId, importId, fileImportId);
Query query = Query.fromQLNullSafe( ql );
- query.setCursor( cursor );
+ query.setOffsetFromCursor( cursor );
query.setConnectionType( FileImportTracker.ERRORS_CONNECTION_NAME );
query.setResultsLevel( Level.ALL_PROPERTIES );
[06/52] [abbrv] incubator-usergrid git commit: Merge branch 'pr/226'
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'pr/226' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a60f1f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a60f1f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a60f1f6
Branch: refs/heads/USERGRID-593
Commit: 5a60f1f641e890c6d2d8968b74718604d2ca1149
Parents: 488fa93 5355864
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 14:13:39 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 14:13:39 2015 -0600
----------------------------------------------------------------------
portal/js/login/login-controller.js | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[44/52] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into workingbranch
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into workingbranch
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0714b110
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0714b110
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0714b110
Branch: refs/heads/USERGRID-593
Commit: 0714b1100929695dd374bed28633e4e31ea80039
Parents: f4d0d1a 5594d74
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Apr 22 14:25:10 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Apr 22 14:25:10 2015 -0600
----------------------------------------------------------------------
.../results/ElasticSearchQueryExecutor.java | 38 +++--
.../org/apache/usergrid/persistence/Query.java | 78 ++++++----
.../apache/usergrid/persistence/Results.java | 65 +++++---
.../usergrid/persistence/CollectionIT.java | 12 +-
.../org/apache/usergrid/persistence/GeoIT.java | 8 +-
.../apache/usergrid/persistence/IndexIT.java | 8 +-
.../PerformanceEntityRebuildIndexTest.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 30 ++--
.../persistence/query/NotSubPropertyIT.java | 2 +-
.../persistence/model/entity/SimpleId.java | 4 +-
.../index/ApplicationEntityIndex.java | 12 +-
.../persistence/index/CandidateResults.java | 39 ++---
.../usergrid/persistence/index/SearchTypes.java | 4 +
.../impl/EsApplicationEntityIndexImpl.java | 155 +++----------------
.../impl/SearchRequestBuilderStrategy.java | 9 +-
.../persistence/index/impl/EntityIndexTest.java | 32 ++--
.../management/importer/ImportServiceImpl.java | 6 +-
19 files changed, 207 insertions(+), 301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0714b110/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
index 018e90b,da0380b..6a45558
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
@@@ -25,7 -25,6 +25,7 @@@ import java.util.UUID
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.model.util.Verify;
- import com.fasterxml.jackson.annotation.JsonTypeInfo;
++
import com.fasterxml.uuid.UUIDComparator;
import com.google.common.base.Preconditions;
@@@ -66,8 -53,13 +65,7 @@@ public class SimpleId implements Id, Se
this(UUIDGenerator.newTimeUUID(), type);
}
- public SimpleId( ){
-
- }
-
-
-
-
@Override
public UUID getUuid() {
return uuid;
[07/52] [abbrv] incubator-usergrid git commit: Merge branch 'pr/202'
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'pr/202' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/945fa084
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/945fa084
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/945fa084
Branch: refs/heads/USERGRID-593
Commit: 945fa084efc2ceb40db7c8ee32b9ddd5ba8eac5a
Parents: 5a60f1f 0c4b7c1
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 14:23:45 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 14:23:45 2015 -0600
----------------------------------------------------------------------
.../usergrid/security/providers/AbstractProvider.java | 6 ++++++
.../security/providers/PingIdentityProvider.java | 14 +++++++++-----
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[31/52] [abbrv] incubator-usergrid git commit: Fixes index runtime
execution for in memory to test
Posted by to...@apache.org.
Fixes index runtime execution for in memory to test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5f99ee2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5f99ee2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5f99ee2d
Branch: refs/heads/USERGRID-593
Commit: 5f99ee2de640093335a73ffcfce3e2b07e8e146c
Parents: 3480a36
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Apr 21 14:12:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Apr 21 14:12:34 2015 -0600
----------------------------------------------------------------------
.../index/AsyncIndexProvider.java | 5 +-
.../index/InMemoryAsyncReIndexService.java | 43 +++++--
.../corepersistence/index/IndexService.java | 12 +-
.../corepersistence/index/IndexServiceImpl.java | 55 ++++++---
.../corepersistence/TestIndexModule.java | 14 ++-
.../corepersistence/index/IndexServiceTest.java | 112 +++++++++++++++++++
.../corepersistence/index/PublishRxtest.java | 39 ++++++-
.../usergrid/persistence/CollectionIT.java | 1 -
.../core/metrics/ObservableTimer.java | 72 ++++++++++++
.../core/rx/RxTaskSchedulerImpl.java | 5 +-
.../persistence/model/entity/Entity.java | 2 +-
.../persistence/index/impl/IndexRequest.java | 1 -
.../usergrid/cassandra/SpringResource.java | 13 ++-
13 files changed, 321 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 2c48c13..18df824 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -43,7 +43,6 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
private final MetricsFactory metricsFactory;
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
- private final AllEntityIdsObservable allEntitiesObservable;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private AsyncReIndexService asyncIndexService;
@@ -53,14 +52,12 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
final MetricsFactory metricsFactory, final IndexService indexService,
final RxTaskScheduler rxTaskScheduler,
- final AllEntityIdsObservable allEntitiesObservable,
final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.queryFig = queryFig;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
this.rxTaskScheduler = rxTaskScheduler;
- this.allEntitiesObservable = allEntitiesObservable;
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -85,7 +82,7 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
switch ( impl ) {
case LOCAL:
return new InMemoryAsyncReIndexService( indexService, rxTaskScheduler,
- entityCollectionManagerFactory );
+ entityCollectionManagerFactory, metricsFactory );
case SQS:
return new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
index 5ebda87..4908945 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
@@ -25,11 +25,14 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -39,33 +42,49 @@ import rx.Observable;
@Singleton
public class InMemoryAsyncReIndexService implements AsyncReIndexService {
- private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncReIndexService.class);
+ private static final Logger log = LoggerFactory.getLogger( InMemoryAsyncReIndexService.class );
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final Timer timer;
@Inject
public InMemoryAsyncReIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ final EntityCollectionManagerFactory entityCollectionManagerFactory, final
+ MetricsFactory metricsFactory ) {
this.indexService = indexService;
this.rxTaskScheduler = rxTaskScheduler;
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+
+ timer = metricsFactory.getTimer( InMemoryAsyncReIndexService.class, "IndexTimer" );
}
@Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity toIndex ) {
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity ) {
//process the entity immediately
//only process the same version, otherwise ignore
- Observable.just( toIndex ).doOnNext( entity -> {
- log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
- indexService.indexEntity( applicationScope, entity );
- } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
- }
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
+
+ final Observable<IndexOperationMessage> edgeObservable = indexService.indexEntity( applicationScope, entity );
+
+
+
+ edgeObservable.subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
+
+ //now start it
+// final Timer.Context time = timer.time();
+//
+// edgeObservable.connect();
+//
+// time.stop();
+
+
+ }
@Override
@@ -75,12 +94,12 @@ public class InMemoryAsyncReIndexService implements AsyncReIndexService {
final Id entityId = entityIdScope.getId();
- final Entity
- entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
- entityId ).toBlocking().lastOrDefault( null );
+ final Entity entity =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope ).load( entityId ).toBlocking()
+ .lastOrDefault( null );
- if(entity == null){
+ if ( entity == null ) {
log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index 41fe323..5e9392b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -20,13 +20,13 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.UUID;
-
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
+import rx.observables.ConnectableObservable;
/**
@@ -41,11 +41,11 @@ public interface IndexService {
* @param applicationScope The scope of the entity
* @param entity The entity
*
- * @return An observable with the count of every batch executed to index the entity. Note that this a cold observable
- * and must be subscribed to in order to perform the operation. This also makes no assumptions on scheduling. It is up to the caller
+ * @return A ConnectableObservable with every edge in the batch to index the entity. Note that this a cold observable
+ * and must be subscribed to, then "connect" in order to perform the operation. This also makes no assumptions on scheduling. It is up to the caller
* to assign the correct scheduler to the observable
*/
- Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity );
+ Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope, final Entity entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 29dd04a..87e22f4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -20,8 +20,11 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.Collection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.graph.Edge;
@@ -38,6 +41,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.InflectionUtils;
+import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -55,26 +59,30 @@ import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
@Singleton
public class IndexServiceImpl implements IndexService {
+
+ private static final Logger logger = LoggerFactory.getLogger( IndexServiceImpl.class );
+
private final GraphManagerFactory graphManagerFactory;
private final EntityIndexFactory entityIndexFactory;
private final EdgesObservable edgesObservable;
private final IndexFig indexFig;
+ private final Timer indexTimer;
@Inject
public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory,
- final EdgesObservable edgesObservable, IndexFig indexFig ) {
+ final EdgesObservable edgesObservable, final IndexFig indexFig, final MetricsFactory metricsFactory ) {
this.graphManagerFactory = graphManagerFactory;
this.entityIndexFactory = entityIndexFactory;
this.edgesObservable = edgesObservable;
this.indexFig = indexFig;
+ this.indexTimer = metricsFactory.getTimer( IndexServiceImpl.class, "index.process");
}
@Override
- public Observable<Long> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
-
-
+ public Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope,
+ final Entity entity ) {
//bootstrap the lower modules from their caches
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
@@ -91,30 +99,37 @@ public class IndexServiceImpl implements IndexService {
//we might or might not need to index from target-> source
+ final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
- final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
+ //merge the edges together
+ final Observable<IndexEdge> observable = Observable.merge( sourceEdgesToIndex, targetSizes);
+ //do our observable for batching
+ //try to send a whole batch if we can
- //start the observable via publish
- final ConnectableObservable<IndexOperationMessage> observable =
- //try to send a whole batch if we can
- Observable.merge( sourceEdgesToIndex, targetSizes ).buffer( indexFig.getIndexBatchSize() )
+ //do our observable for batching
+ //try to send a whole batch if we can
+ final Observable<IndexOperationMessage> batches = observable.buffer( indexFig.getIndexBatchSize() )
- //map into batches based on our buffer size
- .flatMap( buffer -> Observable.from( buffer ).collect( () -> ei.createBatch(),
- ( batch, indexEdge ) -> batch.index( indexEdge, entity ) )
+ //map into batches based on our buffer size
+ .flatMap( buffer -> Observable.from( buffer )
+ //collect results into a single batch
+ .collect( () -> ei.createBatch(), ( batch, indexEdge ) -> {
+ logger.debug( "adding edge {} to batch for entity {}", indexEdge, entity );
+ batch.index( indexEdge, entity );
+ } )
//return the future from the batch execution
- .flatMap( batch -> Observable.from( batch.execute() ) ) ).publish();
+ .flatMap( batch -> Observable.from( batch.execute() ) ) );
+ return ObservableTimer.time( batches, indexTimer );
+ }
- return observable.countLong();
- }
/**
- * Get index edgs to the target
+ * Get index edges to the target
*
* @param graphManager The graph manager
* @param entityId The entitie's id
@@ -147,6 +162,10 @@ public class IndexServiceImpl implements IndexService {
*
* we're indexing from target->source here
*/
- return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection ).map( edge -> generateScopeToTarget( edge ) );
+ return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
+ .map( edge -> generateScopeToTarget( edge ) );
}
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
index e5254d1..95000bf 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
@@ -19,17 +19,25 @@
package org.apache.usergrid.corepersistence;
+import org.springframework.context.ApplicationContext;
+
+import org.apache.usergrid.persistence.PersistenceModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+import org.apache.usergrid.setup.ConcurrentProcessSingleton;
public class TestIndexModule extends TestModule {
+
@Override
protected void configure() {
- //this will break, we need to untagle this and move to guice in core completely
- install( new CoreModule( ));
- }
+ //TODO, refactor to guice to get rid of this
+ final ApplicationContext singleton = ConcurrentProcessSingleton.getInstance().getSpringResource().getAppContext();
+ //this will break, we need to untagle this and move to guice in core completely
+ install( new CoreModule() );
+ install( new PersistenceModule( singleton ) );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
new file mode 100644
index 0000000..b0f3995
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.List;
+import java.util.Set;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.*;
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+public class IndexServiceTest {
+
+ @Inject
+ public IndexService indexService;
+
+
+ @Inject
+ public GraphManagerFactory graphManagerFactory;
+
+ public GraphManager graphManager;
+
+ public ApplicationScope applicationScope;
+
+ @Before
+ public void setup(){
+ applicationScope = getApplicationScope( UUIDGenerator.newTimeUUID());
+
+ graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+ }
+
+
+ @Test
+ public void testSingleIndexFromSource(){
+ final Entity entity = new Entity( createId( "test" ), UUIDGenerator.newTimeUUID());
+ entity.setField( new StringField( "string", "foo" ) );
+
+ final Edge collectionEdge = createCollectionEdge( applicationScope.getApplication(), "tests", entity.getId() );
+
+ //write the edge
+ graphManager.writeEdge( collectionEdge ).toBlocking().last();
+
+
+ //index the edge
+ final Observable<IndexOperationMessage> indexed = indexService.indexEntity( applicationScope, entity );
+
+
+ //real users should never call to blocking, we're not sure what we'll get
+ final IndexOperationMessage results = indexed.toBlocking().last();
+
+ final Set<IndexRequest> indexRequests = results.getIndexRequests();
+
+ //ensure our value made it to the index request
+ final IndexRequest indexRequest = indexRequests.iterator().next();
+
+ assertNotNull(indexRequest);
+
+
+
+// assertEquals(applicationScope.getApplication(), indexRequest.);
+// assertEquals(collectionEdge.getTimestamp(), edge.getTimestamp());
+// assertEquals(collectionEdge.getType(), edge.getEdgeName());
+// assertEquals( SearchEdge.NodeType.TARGET, edge.getNodeType());
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
index 1eb00c2..84d3ef7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.index;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import org.junit.Ignore;
import org.junit.Test;
import rx.Observable;
@@ -30,6 +31,7 @@ import rx.Subscription;
import rx.observables.ConnectableObservable;
import rx.schedulers.Schedulers;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -54,8 +56,41 @@ public class PublishRxTest {
assertTrue( "publish1 behaves as expected", completed );
- final boolean completedSubscription = connectedObservable.isUnsubscribed();;
+ final boolean completedSubscription = connectedObservable.isUnsubscribed();
+ ;
- assertTrue("Subscription complete", completedSubscription);
+ assertTrue( "Subscription complete", completedSubscription );
+ }
+
+
+ @Test
+ @Ignore("This seems like it should work, yet blocks forever")
+ public void testConnectableObserver() throws InterruptedException {
+
+ final int count = 10;
+
+ final CountDownLatch latch = new CountDownLatch( count );
+
+ final ConnectableObservable<Integer> connectedObservable = Observable.range( 0, count ).publish();
+
+
+ //connect to our latch, which should run on it's own subscription
+ //start our latch running
+ connectedObservable.doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ final Observable<Integer> countObservable = connectedObservable.subscribeOn( Schedulers.io() ).count();
+
+ //start the sequence
+ connectedObservable.connect();
+
+
+ final boolean completed = latch.await( 5, TimeUnit.SECONDS );
+
+ assertTrue( "publish1 behaves as expected", completed );
+
+ final int returnedCount = countObservable.toBlocking().last();
+
+ assertEquals( "Counts the same", count, returnedCount );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index d1f746a..fcdc11f 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -314,7 +314,6 @@ public class CollectionIT extends AbstractCoreIT {
assertNotNull( user );
app.refreshIndex();
- Thread.sleep( 100 );
// EntityRef
final Query query = Query.fromQL( "lastname = '" + lastName + "'" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/ObservableTimer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/ObservableTimer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/ObservableTimer.java
new file mode 100644
index 0000000..c8ffeb8
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/ObservableTimer.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.metrics;
+
+
+import com.codahale.metrics.Timer;
+
+import rx.Observable;
+
+
+/**
+ * A wrapper class that will allows timing around an observable. Simply pass an observable and we will set a start and
+ * stop issue.
+ */
+public class ObservableTimer {
+
+
+ private final Timer timer;
+ private Timer.Context context;
+
+
+ /**
+ * Intentionally private, use the factory
+ *
+ * @param timer
+ */
+ private ObservableTimer( final Timer timer ) {this.timer = timer;}
+
+
+ /**
+ * Start the timer
+ */
+ public void start() {
+ context = timer.time();
+ }
+
+
+ /**
+ * Stop the timer
+ */
+ public void stop() {
+ context.stop();
+ }
+
+
+ /**
+ * Time the obserable with the specified timer
+ */
+ public static <T> Observable<T> time( final Observable<T> observable, final Timer timer ) {
+ final ObservableTimer proxy = new ObservableTimer( timer );
+
+ //attach to the observable
+ return observable.doOnSubscribe( () -> proxy.start() ).doOnCompleted( () -> proxy.stop() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
index 311f7f4..f2950f4 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
@@ -100,8 +100,11 @@ public class RxTaskSchedulerImpl implements RxTaskScheduler {
public Thread newThread( final Runnable r ) {
final long newValue = threadCounter.incrementAndGet();
- Thread t = new Thread( r, poolName + "-" + newValue );
+ final String threadName = poolName + "-" + newValue;
+ Thread t = new Thread( r, threadName );
+
+ //set it to be a daemon thread so it doesn't block shutdown
t.setDaemon( true );
return t;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
index 3296203..ae8662f 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
@@ -69,7 +69,7 @@ public class Entity extends EntityObject {
this.id = id;
}
- protected Entity(Id id, UUID version){
+ public Entity(Id id, UUID version){
this(id);
this.version = version;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index a2a1c4d..5852394 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -41,7 +41,6 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
/**
* Represent the properties required to build an index request
*/
-@JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" )
public class IndexRequest implements BatchRequest {
public String writeAlias;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f99ee2d/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/SpringResource.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/SpringResource.java b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/SpringResource.java
index 2370c4b..7d3782d 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/SpringResource.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/SpringResource.java
@@ -17,12 +17,9 @@
package org.apache.usergrid.cassandra;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-import org.safehaus.guicyfig.EnvironResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
@@ -93,6 +90,14 @@ public class SpringResource {
/**
+ * Use this with care. You should use getBean in most situations
+ * @return
+ */
+ public ApplicationContext getAppContext(){
+ return applicationContext;
+ }
+
+ /**
* Gets a bean from the application context.
*
* @param requiredType the type of the bean
[30/52] [abbrv] incubator-usergrid git commit: Added fix to test file
PublishRxtest now PublishRxTest
Posted by to...@apache.org.
Added fix to test file PublishRxtest now PublishRxTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f2aa4032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f2aa4032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f2aa4032
Branch: refs/heads/USERGRID-593
Commit: f2aa4032565eb6295c6568b3fe7049eb14cdcf73
Parents: 3480a36
Author: GERey <gr...@apigee.com>
Authored: Tue Apr 21 12:34:39 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue Apr 21 12:34:39 2015 -0700
----------------------------------------------------------------------
.../corepersistence/index/PublishRxTest.java | 7 +++
.../corepersistence/index/PublishRxtest.java | 61 --------------------
2 files changed, 7 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2aa4032/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
new file mode 100644
index 0000000..1d41286
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxTest.java
@@ -0,0 +1,7 @@
+package org.apache.usergrid.corepersistence.index;
+
+
+/**
+ * Created by ApigeeCorporation on 4/21/15.
+ */
+public class PublishRxTest {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2aa4032/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
deleted file mode 100644
index 1eb00c2..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/PublishRxtest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.junit.Test;
-
-import rx.Observable;
-import rx.Subscription;
-import rx.observables.ConnectableObservable;
-import rx.schedulers.Schedulers;
-
-import static org.junit.Assert.assertTrue;
-
-
-/**
- * Validates the Rx scheduler works as expected with publish
- */
-public class PublishRxTest {
-
- @Test
- public void testPublish() throws InterruptedException {
-
- final int count = 10;
-
- final CountDownLatch latch = new CountDownLatch( count );
-
- final Subscription connectedObservable =
- Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
- .subscribe();
-
-
- final boolean completed = latch.await( 5, TimeUnit.SECONDS );
-
- assertTrue( "publish1 behaves as expected", completed );
-
- final boolean completedSubscription = connectedObservable.isUnsubscribed();;
-
- assertTrue("Subscription complete", completedSubscription);
- }
-}
[18/52] [abbrv] incubator-usergrid git commit: Fixed ownership test
along with adding the graph mechanism to createIteminCollections!
Posted by to...@apache.org.
Fixed ownership test along with adding the graph mechanism to createIteminCollections!
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4cc2e0bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4cc2e0bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4cc2e0bf
Branch: refs/heads/USERGRID-593
Commit: 4cc2e0bfb2cb55833a032d8725a345c1a425a7eb
Parents: 8dafe7b
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 16:21:26 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 16:21:26 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpRelationManager.java | 7 +++++--
.../org/apache/usergrid/persistence/EntityManagerIT.java | 8 ++------
2 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cc2e0bf/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index be8605e..a3d8172 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -505,8 +505,11 @@ public class CpRelationManager implements RelationManager {
addToCollection( collName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
- throw new UnsupportedOperationException( "Implement me directly in graph " );
-// getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
+ Id itemEntityId = new SimpleId( itemEntity.getUuid(),itemEntity.getType() );
+ final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collName, itemEntityId );
+
+ GraphManager gm = managerCache.getGraphManager( applicationScope );
+ gm.writeEdge( edge );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cc2e0bf/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index 3545739..f161040 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -26,19 +26,16 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.RandomStringUtils;
-
import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
@@ -46,7 +43,6 @@ import static org.apache.usergrid.persistence.Schema.PROPERTY_APPLICATION_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
[21/52] [abbrv] incubator-usergrid git commit: Refactored observables
Posted by to...@apache.org.
Refactored observables
Refactored re-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/59455748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/59455748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/59455748
Branch: refs/heads/USERGRID-593
Commit: 59455748c0978212396b4bf8f53bc410af81f922
Parents: 078666e
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 20 18:27:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 20 18:27:19 2015 -0600
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 35 ------
.../usergrid/corepersistence/CoreModule.java | 1 +
.../events/EntityVersionDeletedHandler.java | 4 +-
.../index/AsyncIndexProvider.java | 17 ++-
.../index/AsyncIndexService.java | 5 +-
.../index/InMemoryAsyncIndexService.java | 39 ++++++-
.../corepersistence/index/QueryFig.java | 5 +
.../corepersistence/index/ReIndexService.java | 96 ++++++++++++++++
.../index/ReIndexServiceImpl.java | 114 +++++++++++++++++++
.../index/SQSAsyncIndexService.java | 9 +-
.../rx/impl/AbstractGraphVisitorImpl.java | 60 ++--------
.../rx/impl/AllApplicationsObservable.java | 43 +++++++
.../rx/impl/AllApplicationsObservableImpl.java | 5 +-
.../rx/impl/AllEntitiesInSystemImpl.java | 14 +--
.../rx/impl/AllEntityIdsObservable.java | 51 +++++++++
.../rx/impl/AllEntityIdsObservableImpl.java | 87 ++++++++++++++
.../rx/impl/AllNodesInGraphImpl.java | 11 +-
.../corepersistence/rx/impl/EdgeScope.java | 51 +++++++++
.../corepersistence/util/CpNamingUtils.java | 15 +--
.../util/SerializableMapper.java | 89 +++++++++++++++
.../rx/ApplicationObservableTestIT.java | 3 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../core/rx/RxTaskSchedulerImpl.java | 9 +-
.../core/scope/ApplicationScope.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 ++++++
.../graph/serialization/EdgesObservable.java | 16 ++-
.../serialization/impl/EdgesObservableImpl.java | 23 +++-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 6 +-
.../persistence/index/CandidateResults.java | 2 +-
.../persistence/index/utils/StringUtils.java | 62 ----------
.../usergrid/rest/management/AdminUsersIT.java | 4 +-
32 files changed, 715 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
deleted file mode 100644
index 24f32f3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence;
-
-
-import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.entities.Application;
-
-import rx.Observable;
-
-
-/**
- * Interface for generating an observable of all ApplicationScope
- */
-public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 6ebff53..f49e066 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 22f599e..7a93606 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -23,8 +23,6 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
@@ -103,7 +101,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
//Remove all double indexes
- final Observable<IndexEdge> sourceScopes = edgesObservable.edgesFromSource( gm, entityId ).map(
+ final Observable<IndexEdge> sourceScopes = edgesObservable.edgesFromSourceAscending( gm, entityId ).map(
edge -> generateScopeToTarget( edge ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 77b4990..d00ef8e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.index;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
@@ -41,19 +43,25 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
private final MetricsFactory metricsFactory;
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
+ private final AllEntityIdsObservable allEntitiesObservable;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private AsyncIndexService asyncIndexService;
@Inject
- public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory, final
- MetricsFactory metricsFactory,
- final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
+ final MetricsFactory metricsFactory, final IndexService indexService,
+ final RxTaskScheduler rxTaskScheduler,
+ final AllEntityIdsObservable allEntitiesObservable,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.queryFig = queryFig;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
this.rxTaskScheduler = rxTaskScheduler;
+ this.allEntitiesObservable = allEntitiesObservable;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -76,7 +84,8 @@ public class AsyncIndexProvider implements Provider<AsyncIndexService> {
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncIndexService( indexService, rxTaskScheduler );
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler,
+ entityCollectionManagerFactory );
case SQS:
return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
default:
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
index 06310ae..8b5ced1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -26,13 +26,15 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
/**
* Low level queue service for indexing entities
*/
-public interface AsyncIndexService {
+public interface AsyncIndexService extends ReIndexService.IndexAction {
/**
@@ -43,4 +45,5 @@ public interface AsyncIndexService {
* @param entity The entity to index
*/
void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
index 3e2a271..0efb964 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -20,10 +20,19 @@
package org.apache.usergrid.corepersistence.index;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -33,16 +42,18 @@ import rx.Observable;
@Singleton
public class InMemoryAsyncIndexService implements AsyncIndexService {
+ private static final Logger log = LoggerFactory.getLogger(InMemoryAsyncIndexService.class);
private final IndexService indexService;
private final RxTaskScheduler rxTaskScheduler;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@Inject
- public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler ) {
+ public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.indexService = indexService;
-
-
this.rxTaskScheduler = rxTaskScheduler;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -53,7 +64,29 @@ public class InMemoryAsyncIndexService implements AsyncIndexService {
//only process the same version, otherwise ignore
Observable.just( toIndex ).doOnNext( entity -> {
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
indexService.indexEntity( applicationScope, entity );
} ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
}
+
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+
+ final Id entityId = entityIdScope.getId();
+
+ final Entity
+ entity = entityCollectionManagerFactory.createCollectionManager( applicationScope ).load(
+ entityId ).toBlocking().lastOrDefault( null );
+
+
+ if(entity == null){
+ log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
+ }
+
+ indexService.indexEntity(applicationScope, entity );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
index a7d2450..82ed496 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
@@ -95,4 +95,9 @@ public interface QueryFig extends GuicyFig {
int getIndexQueueSize();
+ @Default("30000")
+ @Key("elasticsearch.reindex.sample.interval")
+ long getReIndexSampleInterval();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
new file mode 100644
index 0000000..dca6cac
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.FutureTask;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.Observer;
+
+
+/**
+ * An interface for re-indexing all entities in an application
+ */
+public interface ReIndexService {
+
+
+ /**
+ * Reindex all applications using the cursor provided
+ *
+ * @param startTimestamp The timestamp to start seeking from
+ *
+ * @return a cursor that can be used to resume the operation on the next run
+ */
+ IndexResponse reIndex( final rx.Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
+ final Optional<Long> startTimestamp, final IndexAction indexAction );
+
+
+ /**
+ * The response when requesting a re-index operation
+ */
+ class IndexResponse {
+ final String cursor;
+ final Observable<Long> indexedEdgecount;
+
+
+ public IndexResponse( final String cursor, final Observable<Long> indexedEdgecount ) {
+ this.cursor = cursor;
+ this.indexedEdgecount = indexedEdgecount;
+ }
+
+
+ /**
+ * Get the cursor used to resume this operation
+ * @return
+ */
+ public String getCursor() {
+ return cursor;
+ }
+
+
+ /**
+ * Return the observable long count of all edges indexed
+ * @return
+ */
+ public Observable<Long> getCount() {
+ return indexedEdgecount;
+ }
+ }
+
+
+
+
+ /**
+ * Callback to perform an index operation based on an scope during bulk re-index operations
+ */
+ @FunctionalInterface
+ interface IndexAction {
+
+ void index( final EntityIdScope entityIdScope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
new file mode 100644
index 0000000..5c022e1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.corepersistence.util.SerializableMapper;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.StringUtils;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.observables.ConnectableObservable;
+
+
+public class ReIndexServiceImpl implements ReIndexService {
+
+ private static final MapScope RESUME_MAP_SCOPTE =
+ new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "reindexresume" );
+
+ //Keep cursors to resume re-index for 1 day. This is far beyond it's useful real world implications anyway.
+ private static final int INDEX_TTL = 60 * 60 * 24 * 10;
+
+
+ private final AllEntityIdsObservable allEntityIdsObservable;
+ private final QueryFig queryFig;
+ private final RxTaskScheduler rxTaskScheduler;
+ private final MapManager mapManager;
+
+
+ public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
+ final MapManagerFactory mapManagerFactory, final QueryFig queryFig,
+ final RxTaskScheduler rxTaskScheduler ) {
+ this.allEntityIdsObservable = allEntityIdsObservable;
+ this.queryFig = queryFig;
+ this.rxTaskScheduler = rxTaskScheduler;
+
+ this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPTE );
+ }
+
+
+ @Override
+ public IndexResponse reIndex( final Observable<ApplicationScope> applicationScopes, final Optional<String> cursor,
+ final Optional<Long> startTimestamp, final IndexAction indexAction ) {
+
+
+ //load our last emitted Scope if a cursor is present
+ if ( cursor.isPresent() ) {
+ throw new UnsupportedOperationException( "Build this" );
+ }
+
+ final String newCursor = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ //create an observable that loads each entity and indexes it, start it running with publish
+ final ConnectableObservable<EdgeScope> runningReIndex =
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, startTimestamp )
+
+ //for each edge, create our scope and index on it
+ .doOnNext( edge -> indexAction
+ .index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) )
+
+ .subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).publish();
+
+
+ //count our longs
+ final Observable<Long> indexedCount = runningReIndex.countLong();
+
+
+ //start our sampler and state persistence
+ //take a sample every sample interval to allow us to resume state with minimal loss
+ runningReIndex.sample( queryFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
+ rxTaskScheduler.getAsyncIOScheduler() )
+ .doOnNext( edge -> {
+
+ final String serializedState = SerializableMapper.asString( edge );
+
+ mapManager.putString( newCursor, serializedState, INDEX_TTL );
+ } ).subscribe();
+
+
+ return new IndexResponse( newCursor, indexedCount );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
index dfcb97a..6d06637 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -233,6 +234,12 @@ public class SQSAsyncIndexService implements AsyncIndexService {
}
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ }
+
+
/**
* The message that subclasses our IndexOperationMessage. holds a pointer to the original message
*/
@@ -258,6 +265,6 @@ public class SQSAsyncIndexService implements AsyncIndexService {
@Override
public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
-
+ throw new UnsupportedOperationException( "Implement me" );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
index ef66481..e307233 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -20,88 +20,46 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
-import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import rx.Observable;
-import rx.functions.Func1;
/**
- * An observable that will emit every entity Id stored in our entire system across all apps.
- * Note that this only walks each application applicationId graph, and emits edges from the applicationId and it's edges as the s
- * source node
+ * An observable that returns all entities in the collections
*/
public abstract class AbstractGraphVisitorImpl<T> implements MigrationDataProvider<T> {
private final AllApplicationsObservable applicationObservable;
- private final GraphManagerFactory graphManagerFactory;
- private final TargetIdObservable targetIdObservable;
+ private final AllEntityIdsObservable allEntityIdsObservable;
@Inject
public AbstractGraphVisitorImpl( AllApplicationsObservable applicationObservable,
- GraphManagerFactory graphManagerFactory, TargetIdObservable targetIdObservable ) {
+ final AllEntityIdsObservable allEntityIdsObservable ) {
this.applicationObservable = applicationObservable;
- this.graphManagerFactory = graphManagerFactory;
- this.targetIdObservable = targetIdObservable;
+ this.allEntityIdsObservable = allEntityIdsObservable;
}
@Override
public Observable<T> getData() {
- return applicationObservable.getData().flatMap( new Func1<ApplicationScope, Observable<T>>() {
- @Override
- public Observable<T> call( final ApplicationScope applicationScope ) {
- return getAllEntities( applicationScope );
- }
- } );
+ return allEntityIdsObservable.getEntities( applicationObservable.getData() ).map(
+ entityIdScope -> generateData( entityIdScope ) );
}
- private Observable<T> getAllEntities(final ApplicationScope applicationScope) {
- final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
- final Id applicationId = applicationScope.getApplication();
-
- //load all nodes that are targets of our application node. I.E.
- // entities that have been saved
- final Observable<Id> entityNodes =
- targetIdObservable.getTargetNodes(gm, applicationId);
-
- //emit Scope + ID
-
- //create our application node to emit since it's an entity as well
- final Observable<Id> applicationNode = Observable.just(applicationId);
-
- //merge both the specified application node and the entity node
- // so they all get used
- return Observable
- .merge( applicationNode, entityNodes ).
- map( new Func1<Id, T>() {
- @Override
- public T call( final Id id ) {
- return generateData(applicationScope, id);
- }
- } );
- }
-
-
/**
* Generate the data for the observable stream from the scope and the node id
- * @param applicationScope
- * @param nodeId
+ * @param entityIdScope
* @return
*/
- protected abstract T generateData(final ApplicationScope applicationScope, final Id nodeId);
+ protected abstract T generateData(final EntityIdScope entityIdScope);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
new file mode 100644
index 0000000..76ad8a1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.amazonaws.services.opsworks.model.App;
+
+import rx.Observable;
+
+
+/**
+ * Interface for generating an observable of all ApplicationScope
+ */
+public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
+
+
+ /**
+ * Get an observable of application scopes to be used
+ * @return
+ */
+ Observable<ApplicationScope> getData();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 5151857..5e2f6b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -23,13 +23,11 @@ package org.apache.usergrid.corepersistence.rx.impl;
import java.util.Arrays;
import java.util.UUID;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -40,6 +38,7 @@ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.utils.UUIDUtils;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
index 83f7d84..a315482 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntitiesInSystemImpl.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,17 +40,14 @@ public class AllEntitiesInSystemImpl extends AbstractGraphVisitorImpl<EntityIdSc
@Inject
public AllEntitiesInSystemImpl( final AllApplicationsObservable applicationObservable,
- final GraphManagerFactory graphManagerFactory,
- final TargetIdObservable targetIdObservable ) {
- super( applicationObservable, graphManagerFactory, targetIdObservable );
+ final AllEntityIdsObservable allEntityIdsObservable) {
+ super( applicationObservable, allEntityIdsObservable );
}
@Override
- protected EntityIdScope generateData( final ApplicationScope applicationScope, final Id nodeId ) {
-
- final EntityIdScope idScope = new EntityIdScope( applicationScope, nodeId );
-
- return idScope;
+ protected EntityIdScope generateData( final EntityIdScope entityIdScope ) {
+ return entityIdScope;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
new file mode 100644
index 0000000..c805a59
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import com.google.common.base.Optional;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import rx.Observable;
+
+
+/**
+ * An implementation that will provide all entityId scopes in the system
+ */
+public interface AllEntityIdsObservable {
+
+ /**
+ * Return an observable of scopes from the given appScopes
+ * @param appScopes
+ * @return An observable of entityId scopes
+ */
+ Observable<EntityIdScope> getEntities( final Observable<ApplicationScope> appScopes );
+
+ /**
+ * Get all edges that represent edges to entities in the system
+ * @param appScopes
+ * @param startTime The time to
+ * @return
+ */
+ Observable<EdgeScope> getEdgesToEntities(final Observable<ApplicationScope> appScopes, final Optional<Long> startTime);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
new file mode 100644
index 0000000..1f74420
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllEntityIdsObservableImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+
+import com.google.common.base.Optional;
+
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * An implementation that will provide all entityId scopes in the system
+ */
+public class AllEntityIdsObservableImpl implements AllEntityIdsObservable {
+ private final GraphManagerFactory graphManagerFactory;
+ private final TargetIdObservable targetIdObservable;
+ private final EdgesObservable edgesObservable;
+
+
+ public AllEntityIdsObservableImpl( final GraphManagerFactory graphManagerFactory,
+ final TargetIdObservable targetIdObservable,
+ final EdgesObservable edgesObservable ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.targetIdObservable = targetIdObservable;
+ this.edgesObservable = edgesObservable;
+ }
+
+
+ @Override
+ public Observable<EntityIdScope> getEntities( final Observable<ApplicationScope> appScopes ) {
+
+ return appScopes.flatMap( applicationScope -> {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final Id applicationId = applicationScope.getApplication();
+
+ //load all nodes that are targets of our application node. I.E.
+ // entities that have been saved
+ final Observable<Id> entityNodes = targetIdObservable.getTargetNodes( gm, applicationId );
+
+
+ //create our application node to emit since it's an entity as well
+ final Observable<Id> applicationNode = Observable.just( applicationId );
+
+ //merge both the specified application node and the entity node
+ // so they all get used
+ return Observable.merge( applicationNode, entityNodes ).
+ map( id -> new EntityIdScope( applicationScope, id ) );
+ } );
+ }
+
+
+ @Override
+ public Observable<EdgeScope> getEdgesToEntities( final Observable<ApplicationScope> appScopes, final Optional<Long> startTime) {
+
+ return appScopes.flatMap( applicationScope -> {
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+
+ return edgesObservable.edgesFromSourceAscending( gm, applicationScope.getApplication(), startTime ).map( edge -> new EdgeScope(applicationScope, edge ));
+ } );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
index 19292b8..f802654 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllNodesInGraphImpl.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.rx.impl;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
@@ -42,14 +42,13 @@ public class AllNodesInGraphImpl extends AbstractGraphVisitorImpl<GraphNode> {
@Inject
public AllNodesInGraphImpl( final AllApplicationsObservable applicationObservable,
- final GraphManagerFactory graphManagerFactory,
- final TargetIdObservable targetIdObservable ) {
- super( applicationObservable, graphManagerFactory, targetIdObservable );
+ final AllEntityIdsObservable allEntityIdsObservable ) {
+ super( applicationObservable, allEntityIdsObservable );
}
@Override
- protected GraphNode generateData( final ApplicationScope applicationScope, final Id nodeId ) {
- return new GraphNode( applicationScope, nodeId );
+ protected GraphNode generateData( final EntityIdScope entityIdScope ) {
+ return new GraphNode(entityIdScope.getApplicationScope(), entityIdScope.getId());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
new file mode 100644
index 0000000..c5639ce
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/EdgeScope.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+
+
+/**
+ * Get the scope for the edge
+ */
+public final class EdgeScope implements Serializable {
+ private final ApplicationScope applicationScope;
+ private final Edge edge;
+
+
+ public EdgeScope( final ApplicationScope applicationScope, final Edge edge ) {
+ this.applicationScope = applicationScope;
+ this.edge = edge;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+ public Edge getEdge() {
+ return edge;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index c60b86e..67cc0ca 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -73,13 +73,6 @@ public class CpNamingUtils {
public static String TYPES_BY_UUID_MAP = "zzz_typesbyuuid_zzz";
- /**
- * Create an Id object from the entity ref
- */
- public static Id createId( final EntityRef entityRef ) {
- return new SimpleId( entityRef.getUuid(), entityRef.getType() );
- }
-
/**
* Generate a standard edge name for our graph using the connection name. To be used only for searching. DO NOT use
@@ -229,6 +222,14 @@ public class CpNamingUtils {
/**
+ * Generate an application scope for the management application
+ * @return
+ */
+ public static Id getManagementApplicationId(){
+ return generateApplicationId( MANAGEMENT_APPLICATION_ID );
+ }
+
+ /**
* Get the map scope for the applicationId to store entity uuid to type mapping
*/
public static MapScope getEntityTypeMapScope( final Id applicationId ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
new file mode 100644
index 0000000..1ad4115
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.util;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Preconditions;
+
+
+/**
+ * A simple utility for serializing serializable classes to/and from strings. To be used for small object storage only, such as resume on re-index
+ * storing data such as entities should be specialized.
+ */
+public class SerializableMapper {
+
+ private static final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
+ static{
+ MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ }
+
+ /**
+ * Get value as a string
+ * @param toSerialize
+ * @param <T>
+ * @return
+ */
+ public static <T extends Serializable> String asString(final T toSerialize){
+ try {
+ return MAPPER.writeValueAsString( toSerialize );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new RuntimeException( "Unable to process json", e );
+ }
+ }
+
+
+ /**
+ * Write the value as a string
+ * @param serialized
+ * @param clazz
+ * @param <T>
+ * @return
+ */
+ public static <T extends Serializable> T fromString(final String serialized, final Class<T> clazz){
+ Preconditions.checkNotNull(serialized, "serialized string cannot be null");
+
+
+ InputStream stream = new ByteArrayInputStream(serialized.getBytes( StandardCharsets.UTF_8));
+
+ try {
+ return MAPPER.readValue( stream, clazz );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( String.format("Unable to parse string '%s'", serialized), e );
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
index 353e2fa..a619e4d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
@@ -28,8 +28,8 @@ import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.cassandra.SpringResource;
-import org.apache.usergrid.corepersistence.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
@@ -52,6 +52,7 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
final Application createdApplication = app.getEntityManager().getApplication();
+
AllApplicationsObservable applicationObservable =SpringResource.getInstance().getBean(Injector.class).getInstance(AllApplicationsObservable.class);
//now our get all apps we expect. There may be more, but we don't care about those.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
index 6d228b2..92f2b01 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.corepersistence.rx;
import java.util.HashSet;
import java.util.Set;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.junit.Test;
@@ -35,15 +34,12 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.utils.EdgeTestUtils;
import com.google.inject.Injector;
-import rx.functions.Action1;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -93,7 +89,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesFromSourceObservable.edgesFromSource( gm, applicationId ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceAscending( gm, applicationId ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
@@ -122,7 +118,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
//test connections
- edgesFromSourceObservable.edgesFromSource( gm, source).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceAscending( gm, source ).doOnNext( edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
index 219cde6..311f7f4 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/RxTaskSchedulerImpl.java
@@ -83,8 +83,7 @@ public class RxTaskSchedulerImpl implements RxTaskScheduler {
public MaxSizeThreadPool( final BlockingQueue<Runnable> queue, final int maxPoolSize ) {
- super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ),
- new RejectedHandler() );
+ super( 1, maxPoolSize, 30, TimeUnit.SECONDS, queue, new CountingThreadFactory( ), new RejectedHandler() );
}
}
@@ -118,11 +117,11 @@ public class RxTaskSchedulerImpl implements RxTaskScheduler {
@Override
public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- log.warn( "{} task queue full, rejecting task {}", poolName, r );
+ log.warn( "{} task queue full, rejecting task {} and running in thread {}", poolName, r, Thread.currentThread().getName() );
- //TODO T.N. do we want to run this on the caller thread?
+ //We've decided we want to have a "caller runs" policy, to just invoke the task when rejected
- throw new RejectedExecutionException( "Unable to run task, queue full" );
+ r.run();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
index 920421d..5d22eff 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/scope/ApplicationScope.java
@@ -19,13 +19,15 @@
package org.apache.usergrid.persistence.core.scope;
+import java.io.Serializable;
+
import org.apache.usergrid.persistence.model.entity.Id;
/**
* A scope used for organizations
*/
-public interface ApplicationScope {
+public interface ApplicationScope extends Serializable {
/**
* Get an Application scope
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
new file mode 100644
index 0000000..acfc2d8
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/util/StringUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.core.util;
+
+
+import java.util.UUID;
+
+
+public class StringUtils extends org.apache.commons.lang.StringUtils {
+
+
+ /**
+ * Remove dashes from our uuid
+ * @param uuid
+ * @return
+ */
+ public static String sanitizeUUID(final UUID uuid){
+ return uuid.toString().replace( "-", "" );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index fa73991..9e7b8e6 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -22,6 +22,9 @@ package org.apache.usergrid.persistence.graph.serialization;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
import rx.Observable;
/**
@@ -35,7 +38,18 @@ public interface EdgesObservable {
* @param sourceNode
* @return
*/
- Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode);
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode );
+
+
+ /**
+ * Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
+ * @param gm
+ * @param sourceNode
+ * @param startTimestamp
+ * @return
+ */
+ Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ final Optional<Long> startTimestamp );
/**
* Get all edges from the source node with the target type
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 2264cbd..859ca2e 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -32,6 +32,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Func1;
@@ -53,7 +55,7 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode ) {
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode ) {
final Observable<String> edgeTypes =
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
@@ -69,6 +71,25 @@ public class EdgesObservableImpl implements EdgesObservable {
@Override
+ public Observable<Edge> edgesFromSourceAscending( final GraphManager gm, final Id sourceNode,
+ final Optional<Long> startTimestamp ) {
+
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+
+
+ return edgeTypes.flatMap( edgeType -> {
+
+ logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
+
+ return gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( sourceNode, edgeType, startTimestamp.or( Long.MIN_VALUE ), SearchByEdgeType.Order.ASCENDING,
+ null ) );
+ } );
+ }
+
+
+ @Override
public Observable<Edge> getEdgesFromSource( final GraphManager gm, final Id sourceNode, final String targetType ) {
final Observable<String> edgeTypes =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 9729e63..5cf5117 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -55,7 +55,7 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSource(gm, sourceNode ).map( new Func1<Edge, Id>() {
+ return edgesFromSourceObservable.edgesFromSourceAscending( gm, sourceNode ).map( new Func1<Edge, Id>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 2e0412f..0df26ff 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -38,8 +38,6 @@ import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSeri
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
import rx.schedulers.Schedulers;
import java.util.List;
@@ -87,9 +85,9 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
-
+
//get edges from the source
- return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
+ return edgesFromSourceObservable.edgesFromSourceAscending( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
final MutationBatch batch = keyspace.prepareMutationBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 14130bc..e4624a9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
+import static org.apache.usergrid.persistence.core.util.StringUtils.sanitizeUUID;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
deleted file mode 100644
index a567594..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.index.utils;
-
-
-import java.util.Arrays;
-import java.util.UUID;
-
-import org.apache.commons.io.IOUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.usergrid.persistence.index.utils.ConversionUtils.string;
-
-
-public class StringUtils extends org.apache.commons.lang.StringUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger( StringUtils.class );
-
-
-
- public static String stringOrSubstringBeforeFirst( String str, char c ) {
- if ( str == null ) {
- return null;
- }
- int i = str.indexOf( c );
- if ( i != -1 ) {
- return str.substring( 0, i );
- }
- return str;
- }
-
-
- public static String toString( Object obj ) {
- return string( obj );
- }
-
-
- /**
- * Remove dashes from our uuid
- * @param uuid
- * @return
- */
- public static String sanitizeUUID(final UUID uuid){
- return uuid.toString().replace( "-", "" );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/59455748/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
index 36a9400..de3a6ce 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
@@ -29,14 +29,13 @@ import javax.mail.internet.MimeMultipart;
import org.junit.Before;
import org.junit.Ignore;
-import org.junit.Rule;
import org.junit.Test;
import org.jvnet.mock_javamail.Mailbox;
import org.apache.usergrid.management.MockImapClient;
-import org.apache.usergrid.persistence.index.utils.StringUtils;
+import org.apache.usergrid.persistence.core.util.StringUtils;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
import org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt.ManagementResource;
@@ -46,7 +45,6 @@ import org.apache.usergrid.rest.test.resource2point0.model.Entity;
import org.apache.usergrid.rest.test.resource2point0.model.QueryParameters;
import org.apache.usergrid.rest.test.resource2point0.model.Token;
-import com.fasterxml.jackson.databind.JsonNode;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.representation.Form;
[34/52] [abbrv] incubator-usergrid git commit: null reference
Posted by to...@apache.org.
null reference
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/587e4393
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/587e4393
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/587e4393
Branch: refs/heads/USERGRID-593
Commit: 587e4393b3d2268d9059877858ac8600bf74318b
Parents: 02defc1
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 14:33:02 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 14:33:02 2015 -0600
----------------------------------------------------------------------
.../corepersistence/results/ElasticSearchQueryExecutor.java | 4 ++--
.../src/main/java/org/apache/usergrid/persistence/Query.java | 3 +++
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/587e4393/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index 1a47717..ca16bc0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -162,10 +162,10 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//from legacy code with no QL set. An empty query is functionally equivalent to select all with default
//sort ordering
- return entityIndex.search( indexScope, types, queryToExecute, query.getLimit() );
+ return entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get());
}
- return entityIndex.search( indexScope, types, queryToExecute, query.getLimit(),cursor.get() );
+ return entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/587e4393/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 02a80af..9f0baf3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -98,6 +98,7 @@ public class Query {
List<Operand> filterClauses = new ArrayList<Operand>();
public Query() {
+ offset = Optional.absent();
}
@@ -273,6 +274,8 @@ public class Query {
if ( cursor != null) {
q = newQueryIfNull( q );
q.setOffsetFromCursor(cursor);
+ }else{
+ q.offset = Optional.absent();
}
if ( limit != null ) {
[03/52] [abbrv] incubator-usergrid git commit: Initial pass at moving
queues to core
Posted by to...@apache.org.
Initial pass at moving queues to core
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/84d779fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/84d779fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/84d779fc
Branch: refs/heads/USERGRID-593
Commit: 84d779fc323b0f6c130c03a8982a39f5d5025b17
Parents: 8bb8a4f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Apr 15 14:21:29 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 11:58:28 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 8 +
.../usergrid/corepersistence/CoreModule.java | 44 ++-
.../corepersistence/CpEntityManager.java | 167 +++++-----
.../corepersistence/CpEntityManagerFactory.java | 18 +-
.../corepersistence/CpManagerCache.java | 3 +
.../corepersistence/CpRelationManager.java | 102 +++----
.../CpRelationManagerFactory.java | 46 ---
.../usergrid/corepersistence/GuiceFactory.java | 19 +-
.../events/EntityDeletedHandler.java | 5 +-
.../events/EntityVersionCreatedHandler.java | 8 +-
.../events/EntityVersionDeletedHandler.java | 19 +-
.../corepersistence/index/BufferQueue.java | 68 +++++
.../index/BufferQueueInMemoryImpl.java | 116 +++++++
.../index/BufferQueueSQSImpl.java | 306 +++++++++++++++++++
.../index/IndexQueueService.java | 45 +++
.../corepersistence/index/IndexService.java | 50 +++
.../corepersistence/index/IndexServiceImpl.java | 148 +++++++++
.../corepersistence/index/QueryFig.java | 98 ++++++
.../corepersistence/index/QueueProvider.java | 112 +++++++
.../index/SQSIndexQueueServiceImpl.java | 35 +++
.../corepersistence/util/CpNamingUtils.java | 4 +-
.../usergrid/persistence/EntityManager.java | 2 -
.../persistence/GuiceAdapterBeanFactory.java | 114 +++++++
.../usergrid/persistence/PersistenceModule.java | 162 ++++++++++
.../corepersistence/TestIndexModule.java | 35 +++
.../index/BufferQueueSQSImplTest.java | 179 +++++++++++
.../persistence/core/aws/NoAWSCredsRule.java | 98 ++++++
.../graph/serialization/EdgesObservable.java | 23 ++
.../serialization/impl/EdgesObservableImpl.java | 68 +++--
.../persistence/index/EntityIndexBatch.java | 3 +-
.../usergrid/persistence/index/IndexFig.java | 104 ++-----
.../persistence/index/guice/IndexModule.java | 5 +-
.../persistence/index/guice/QueueProvider.java | 116 -------
.../persistence/index/impl/BufferQueue.java | 66 ----
.../index/impl/BufferQueueInMemoryImpl.java | 115 -------
.../index/impl/BufferQueueSQSImpl.java | 306 -------------------
.../impl/EsApplicationEntityIndexImpl.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 16 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 50 +--
.../index/impl/EsIndexBufferConsumerImpl.java | 237 ++++++--------
.../index/impl/EsIndexBufferProducerImpl.java | 59 ----
.../index/impl/FailureMonitorImpl.java | 26 --
.../index/impl/FlushBufferQueue.java | 23 ++
.../index/impl/IndexBufferConsumer.java | 15 +-
.../index/impl/IndexBufferProducer.java | 32 --
.../persistence/index/impl/IndexIdentifier.java | 46 +++
.../index/impl/IndexIdentifierImpl.java | 118 +------
.../index/impl/IndexOperationMessage.java | 139 +++++++++
.../index/impl/IndexRefreshCommandImpl.java | 12 +-
.../index/migration/LegacyIndexIdentifier.java | 4 +-
.../index/guice/TestIndexModule.java | 22 +-
.../index/impl/BufferQueueSQSImplTest.java | 175 -----------
.../persistence/queue/NoAWSCredsRule.java | 98 ------
.../persistence/queue/QueueManagerTest.java | 3 +-
stack/services/pom.xml | 50 +--
.../notifications/NotifiersServiceIT.java | 19 +-
57 files changed, 2232 insertions(+), 1749 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 6f3c381..3504be4 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -163,6 +163,14 @@
<dependencies>
+
+ <!-- dependent on wiring guice and spring -->
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-spring</artifactId>
+ <version>4.0-beta5</version>
+ </dependency>
+
<!-- Apache Dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7859ffc..7e8af87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,7 +16,13 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.corepersistence.index.BufferQueue;
+import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.QueryFig;
+import org.apache.usergrid.corepersistence.index.QueueProvider;
import org.apache.usergrid.corepersistence.migration.*;
+import org.apache.usergrid.persistence.PersistenceModule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.safehaus.guicyfig.GuicyFigModule;
import org.springframework.context.ApplicationContext;
@@ -56,25 +62,18 @@ import com.google.inject.multibindings.Multibinder;
*/
public class CoreModule extends AbstractModule {
- /**
- * TODO this is a circular dependency, and should be refactored
- */
- private LazyEntityManagerFactoryProvider lazyEntityManagerFactoryProvider;
- public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
+ public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
- public CoreModule( final ApplicationContext context ) {
- this.lazyEntityManagerFactoryProvider = new LazyEntityManagerFactoryProvider( context );
- }
@Override
protected void configure() {
- //See TODO, this is fugly
- bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
+// //See TODO, this is fugly
+// bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
install( new CommonModule());
install( new CollectionModule() {
@@ -141,31 +140,24 @@ public class CoreModule extends AbstractModule {
plugins.addBinding().to( AppInfoMigrationPlugin.class );
plugins.addBinding().to( MigrationModuleVersionPlugin.class );
- bind(AllApplicationsObservable.class).to(AllApplicationsObservableImpl.class);
+ bind( AllApplicationsObservable.class ).to( AllApplicationsObservableImpl.class );
- install(new GuicyFigModule(ApplicationIdCacheFig.class));
-
- }
+ /*****
+ * Indexing service
+ *****/
- /**
- * TODO, this is a hack workaround due to the guice/spring EMF circular dependency
- * Once the entity managers have been refactored and moved into guice, remove this dependency.
- *
- */
- public static class LazyEntityManagerFactoryProvider implements Provider<EntityManagerFactory>{
- private final ApplicationContext context;
+ bind(IndexService.class).to(IndexServiceImpl.class);
+ //bind the queue provider
+ bind( BufferQueue.class).toProvider( QueueProvider.class );
- public LazyEntityManagerFactoryProvider( final ApplicationContext context ) {this.context = context;}
+ install( new GuicyFigModule( QueryFig.class ) );
+ install( new GuicyFigModule( ApplicationIdCacheFig.class ) );
- @Override
- public EntityManagerFactory get() {
- return this.context.getBean( EntityManagerFactory.class );
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2e7b7d8..a615a43 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -80,9 +80,6 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.map.MapManager;
@@ -125,7 +122,6 @@ import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
@@ -172,7 +168,6 @@ public class CpEntityManager implements EntityManager {
private UUID applicationId;
private Application application;
- private CpEntityManagerFactory emf;
private ManagerCache managerCache;
@@ -212,29 +207,34 @@ public class CpEntityManager implements EntityManager {
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
- public CpEntityManager() {
-
- }
-
- @Override
- public void init( EntityManagerFactory emf, UUID applicationId ) {
+ /**
+ * Fugly, make this part of DI
+ * @param cass
+ * @param counterUtils
+ * @param managerCache
+ * @param metricsFactory
+ * @param applicationId
+ */
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
- Preconditions.checkNotNull( emf, "emf must not be null" );
+ Preconditions.checkNotNull( cass, "cass must not be null" );
+ Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
+ Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
- this.emf = ( CpEntityManagerFactory ) emf;
- this.managerCache = this.emf.getManagerCache();
+
+ this.managerCache = managerCache;
this.applicationId = applicationId;
applicationScope = CpNamingUtils.getApplicationScope( applicationId );
ecm = managerCache.getEntityCollectionManager( applicationScope );
- this.cass = this.emf.getCassandraService();
- this.counterUtils = this.emf.getCounterUtils();
+ this.cass = cass;
+ this.counterUtils = counterUtils;
//Timer Setup
- this.metricsFactory = this.emf.getMetricsFactory();
+ this.metricsFactory = metricsFactory;
this.aggCounterTimer =this.metricsFactory.getTimer( CpEntityManager.class,
"cp.entity.get.aggregate.counters.timer" );
this.entCreateTimer =this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.create.timer" );
@@ -724,18 +724,18 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Map<String, Object> properties ) throws Exception {
- this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties);
+ this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties );
this.application = get( applicationId, Application.class );
}
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
- Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
- CpRelationManager rmi = CpRelationManagerFactory.get(
- this, emf, applicationId, entityRef, null, metricsFactory
- );
- return rmi;
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
+
+ CpRelationManager relationManager =
+ new CpRelationManager( metricsFactory, managerCache, this, applicationId, entityRef );
+ return relationManager;
}
@@ -819,7 +819,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
- return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias);
+ return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
}
@@ -848,7 +848,7 @@ public class CpEntityManager implements EntityManager {
new Object[] { ownerRef, collectionType, aliasValue } );
}
- return results.get(aliasValue);
+ return results.get( aliasValue );
}
@@ -1226,7 +1226,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF dictionaryCf = null;
- boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName);
+ boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName );
if ( entityHasDictionary ) {
dictionaryCf = ENTITY_DICTIONARIES;
@@ -1279,7 +1279,7 @@ public class CpEntityManager implements EntityManager {
Class<?> dictionaryCoType =
Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
- boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType);
+ boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
for ( int i = 0; i < elementNames.length; i++ ) {
@@ -1349,14 +1349,14 @@ public class CpEntityManager implements EntityManager {
@Override
public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isCollectionMember(collectionName, entity);
+ return getRelationManager( owner ).isCollectionMember( collectionName, entity );
}
@Override
public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isConnectionMember(connectionName, entity);
+ return getRelationManager( owner ).isConnectionMember( connectionName, entity );
}
@@ -1455,7 +1455,7 @@ public class CpEntityManager implements EntityManager {
public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
- return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef);
+ return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef );
}
@@ -1465,7 +1465,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( connectingEntity )
- .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
+ .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
}
@@ -1523,7 +1523,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities(connectionType, connectedEntityType, resultsLevel);
+ .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
}
@@ -1532,7 +1532,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities(connectionType, connectedEntityType, resultsLevel);
+ .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
}
@@ -1540,21 +1540,21 @@ public class CpEntityManager implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String entityType, Level level, int count ) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count);
+ return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
}
@Override
public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities(query);
+ return getRelationManager( connectingEntity ).searchConnectedEntities( query );
}
@Override
public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception {
- return getRelationManager( entity ).getConnectionIndexes(connectionType);
+ return getRelationManager( entity ).getConnectionIndexes( connectionType );
}
@@ -1799,8 +1799,8 @@ public class CpEntityManager implements EntityManager {
permission = permission.toLowerCase();
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp);
+ CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
@@ -1816,7 +1816,7 @@ public class CpEntityManager implements EntityManager {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, timestamp );
+ getRolePermissionsKey( groupId, roleName ), permission, timestamp );
//Adding graphite metrics
Timer.Context timeRevokeGroupRolePermission = entRevokeGroupPermissionTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -2281,7 +2281,7 @@ public class CpEntityManager implements EntityManager {
private MapManager getMapManagerForTypes() {
Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
MapManager mm = managerCache.getMapManager(ms);
@@ -2836,26 +2836,25 @@ public class CpEntityManager implements EntityManager {
CpWalker walker = new CpWalker( );
- walker.walkCollections(
- this, getApplication(), collectionName, reverse, new CpVisitor() {
+ walker.walkCollections( this, getApplication(), collectionName, reverse, new CpVisitor() {
- @Override
- public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
+ @Override
+ public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- // swallow this, it just means this was already updated, which accomplishes our task
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
- entity.getName(), collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
+ try {
+ em.update( entity );
+ po.onProgress( entity );
+ }
+ catch ( WriteOptimisticVerifyException wo ) {
+ // swallow this, it just means this was already updated, which accomplishes our task
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error repersisting entity", ex );
+ }
}
- }
- } );
+ } );
}
@@ -2890,32 +2889,34 @@ public class CpEntityManager implements EntityManager {
void indexEntityIntoCollection( final Edge edge, final org.apache.usergrid.persistence.model.entity.Entity collectionMember) {
- final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
- final EntityIndexBatch batch = aie.createBatch();
-
- // index member into entity collection | type scope
- final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
- batch.index( collectionIndexScope, collectionMember );
-
- //TODO REMOVE INDEX CODE
- // // index member into entity | all-types scope
- // IndexScope entityAllTypesScope = new IndexScopeImpl(
- // collectionEntity.getId(),
- // CpNamingUtils.ALL_TYPES, entityType );
- //
- // batch.index(entityAllTypesScope, memberEntity);
- //
- // // index member into application | all-types scope
- // IndexScope appAllTypesScope = new IndexScopeImpl(
- // getApplicationScope().getApplication(),
- // CpNamingUtils.ALL_TYPES, entityType );
- //
- // batch.index(appAllTypesScope, memberEntity);
-
- //Adding graphite metrics
- Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
- batch.execute();
- timeIndexEntityCollection.stop();
+ throw new UnsupportedOperationException( "Use the new interface!" );
+//
+// final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
+// final EntityIndexBatch batch = aie.createBatch();
+//
+// // index member into entity collection | type scope
+// final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
+// batch.index( collectionIndexScope, collectionMember );
+//
+// //TODO REMOVE INDEX CODE
+// // // index member into entity | all-types scope
+// // IndexScope entityAllTypesScope = new IndexScopeImpl(
+// // collectionEntity.getId(),
+// // CpNamingUtils.ALL_TYPES, entityType );
+// //
+// // batch.index(entityAllTypesScope, memberEntity);
+// //
+// // // index member into application | all-types scope
+// // IndexScope appAllTypesScope = new IndexScopeImpl(
+// // getApplicationScope().getApplication(),
+// // CpNamingUtils.ALL_TYPES, entityType );
+// //
+// // batch.index(appAllTypesScope, memberEntity);
+//
+// //Adding graphite metrics
+// Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
+// batch.execute();
+// timeIndexEntityCollection.stop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9907f91..e03ed47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -156,14 +156,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public ManagerCache getManagerCache() {
-
- if ( managerCache == null ) {
- managerCache = injector.getInstance( ManagerCache.class );
- }
- return managerCache;
- }
-
private Observable<EntityIdScope> getAllEntitiesObservable(){
return injector.getInstance( Key.get(new TypeLiteral< MigrationDataProvider<EntityIdScope>>(){})).getData();
}
@@ -184,16 +176,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager();
- em.init( this ,applicationId );
-
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, managerCache, metricsFactory, applicationId );
return em;
}
- public MetricsFactory getMetricsFactory(){
- return metricsFactory;
- }
-
@Override
public Entity createApplicationV2(String organizationName, String name) throws Exception {
return createApplicationV2( organizationName, name, null );
@@ -746,7 +732,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public Health getEntityStoreHealth() {
// could use any collection scope here, does not matter
- EntityCollectionManager ecm = getManagerCache().getEntityCollectionManager(
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager(
new ApplicationScopeImpl( new SimpleId( CpNamingUtils.MANAGEMENT_APPLICATION_ID, "application" ) ) );
return ecm.getHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index 4cae31e..f4fee0c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -30,6 +30,9 @@ import org.apache.usergrid.persistence.map.MapScope;
import com.google.inject.Inject;
+/**
+ * Cache for managing our other managers. Now just a delegate. Needs refactored away
+ */
public class CpManagerCache implements ManagerCache {
private final EntityCollectionManagerFactory ecmf;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1ae1db1..3c72b60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -107,9 +107,6 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
-
- private CpEntityManagerFactory emf;
-
private ManagerCache managerCache;
private EntityManager em;
@@ -126,25 +123,18 @@ public class CpRelationManager implements RelationManager {
private Timer updateCollectionTimer;
- public CpRelationManager() {}
-
-
- public CpRelationManager init( EntityManager em, CpEntityManagerFactory emf, UUID applicationId,
- EntityRef headEntity, IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory ) {
+ public CpRelationManager(final MetricsFactory metricsFactory, final ManagerCache managerCache, final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
Assert.notNull( em, "Entity manager cannot be null" );
- Assert.notNull( emf, "Entity manager factory cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
Assert.notNull( headEntity, "Head entity cannot be null" );
Assert.notNull( headEntity.getUuid(), "Head entity uuid cannot be null" );
// TODO: this assert should not be failing
//Assert.notNull( indexBucketLocator, "indexBucketLocator cannot be null" );
this.em = em;
- this.emf = emf;
this.applicationId = applicationId;
this.headEntity = headEntity;
- this.managerCache = emf.getManagerCache();
+ this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
this.metricsFactory = metricsFactory;
@@ -165,7 +155,7 @@ public class CpRelationManager implements RelationManager {
Assert.notNull( cpHeadEntity, String
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
- return this;
+
}
@@ -260,40 +250,42 @@ public class CpRelationManager implements RelationManager {
final org.apache.usergrid.persistence.model.entity.Entity cpEntity ) {
- final GraphManager gm = managerCache.getGraphManager( applicationScope );
-
- // loop through all types of edge to target
-
-
- final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
-
- final EntityIndexBatch entityIndexBatch = ei.createBatch();
-
- final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
-
- // for each edge type, emit all the edges of that type
- .flatMap( etype -> gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) ) )
-
- //for each edge we receive index and add to the batch
- .doOnNext( edge -> {
- // reindex the entity in the source entity's collection or connection index
-
- IndexEdge indexScope = generateScopeFromSource( edge );
-
- entityIndexBatch.index( indexScope, cpEntity );
-
- } ).doOnCompleted( () -> {
- Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
- entityIndexBatch.execute();
- timeElasticIndexBatch.stop();
- } ).count().toBlocking().lastOrDefault( 0 );
-
- //Adding graphite metrics
-
-
- logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
+ throw new UnsupportedOperationException( "Use the new interface" );
+
+// final GraphManager gm = managerCache.getGraphManager( applicationScope );
+//
+// // loop through all types of edge to target
+//
+//
+// final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+//
+// final EntityIndexBatch entityIndexBatch = ei.createBatch();
+//
+// final int count = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) )
+//
+// // for each edge type, emit all the edges of that type
+// .flatMap( etype -> gm.loadEdgesToTarget(
+// new SimpleSearchByEdgeType( cpHeadEntity.getId(), etype, Long.MAX_VALUE,
+// SearchByEdgeType.Order.DESCENDING, null ) ) )
+//
+// //for each edge we receive index and add to the batch
+// .doOnNext( edge -> {
+// // reindex the entity in the source entity's collection or connection index
+//
+// IndexEdge indexScope = generateScopeFromSource( edge );
+//
+// entityIndexBatch.index( indexScope, cpEntity );
+//
+// } ).doOnCompleted( () -> {
+// Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
+// entityIndexBatch.execute();
+// timeElasticIndexBatch.stop();
+// } ).count().toBlocking().lastOrDefault( 0 );
+//
+// //Adding graphite metrics
+//
+//
+// logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
}
@@ -494,9 +486,10 @@ public class CpRelationManager implements RelationManager {
// headEntityScope.getName()});
if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
- getRelationManager( itemEntity )
- .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
- getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
+ throw new UnsupportedOperationException( "Implement me directly in graph " );
+// getRelationManager( itemEntity )
+// .addToCollection( collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
+// getRelationManager( itemEntity ).addToCollection( collection.getLinkedCollection(), headEntity, false );
}
return itemEntity;
@@ -555,7 +548,8 @@ public class CpRelationManager implements RelationManager {
addToCollection( collName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
- getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
+ throw new UnsupportedOperationException( "Implement me directly in graph " );
+// getRelationManager( getHeadEntity() ).addToCollection( collection.getLinkedCollection(), itemEntity );
}
}
@@ -1045,12 +1039,6 @@ public class CpRelationManager implements RelationManager {
}
- private CpRelationManager getRelationManager( EntityRef headEntity ) {
- CpRelationManager rmi = new CpRelationManager();
- rmi.init( em, emf, applicationId, headEntity, null, metricsFactory );
- return rmi;
- }
-
/** side effect: converts headEntity into an Entity if it is an EntityRef! */
private Entity getHeadEntity() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
deleted file mode 100644
index 4223f37..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManagerFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.corepersistence;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-
-import java.util.UUID;
-
-/**
- * Factory to return and init relation manager instances
- */
-public class CpRelationManagerFactory {
-
-
- public static CpRelationManager get( EntityManager em,
- CpEntityManagerFactory emf,
- UUID applicationId,
- EntityRef headEntity,
- IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory){
- CpRelationManager relationManager = new CpRelationManager();
- relationManager.init(em,emf,applicationId,headEntity,indexBucketLocator,metricsFactory);
- return relationManager;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
index 566430f..3a08034 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceFactory.java
@@ -31,8 +31,7 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.PersistenceModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -47,7 +46,7 @@ import me.prettyprint.cassandra.service.CassandraHostConfigurator;
* Factory for configuring Guice then returning it
*/
@Singleton
-public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAware {
+public class GuiceFactory implements FactoryBean<Injector> {
private static final Logger logger = LoggerFactory.getLogger( GuiceFactory.class );
@@ -55,16 +54,16 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
private final Properties systemProperties;
-
private ApplicationContext applicationContext;
private Injector injector;
- public GuiceFactory( final CassandraHostConfigurator chc, final Properties systemProperties ) {
+ public GuiceFactory( final ApplicationContext applicationContext, final CassandraHostConfigurator chc, final Properties systemProperties ) {
this.chc = chc;
this.systemProperties = systemProperties;
+ this.applicationContext = applicationContext;
}
@@ -128,8 +127,10 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
throw new RuntimeException( "Fatal error loading configuration.", e );
}
- //this is seriously fugly, and needs removed
- injector = Guice.createInjector( new CoreModule( applicationContext ) );
+
+
+ //this is seriously fugly, and needs removed we shouldn't be mixing spring and guice
+ injector = Guice.createInjector( new CoreModule( ), new PersistenceModule( applicationContext ) );
return injector;
}
@@ -147,8 +148,4 @@ public class GuiceFactory implements FactoryBean<Injector>, ApplicationContextAw
}
- @Override
- public void setApplicationContext( final ApplicationContext applicationContext ) throws BeansException {
- this.applicationContext = applicationContext;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 6a37144..2de69ed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -42,11 +42,8 @@ public class EntityDeletedHandler implements EntityDeleted {
private static final Logger logger = LoggerFactory.getLogger( EntityDeletedHandler.class );
- private final EntityManagerFactory emf;
-
-
@Inject
- public EntityDeletedHandler( final EntityManagerFactory emf ) {this.emf = emf;}
+ public EntityDeletedHandler( ) {}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 0163fc2..20bdd55 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -41,15 +41,11 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
- private final EntityManagerFactory emf;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@Inject
- public EntityVersionCreatedHandler( final EntityManagerFactory emf,
- final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.emf = emf;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ public EntityVersionCreatedHandler( ) {
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 700851a..22f599e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -30,9 +30,11 @@ import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -56,17 +58,20 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private static final Logger logger = LoggerFactory.getLogger( EntityVersionDeletedHandler.class );
- private final EntityManagerFactory emf;
private final EdgesObservable edgesObservable;
private final SerializationFig serializationFig;
+ private final EntityIndexFactory entityIndexFactory;
+ private final GraphManagerFactory graphManagerFactory;
@Inject
- public EntityVersionDeletedHandler( final EntityManagerFactory emf, final EdgesObservable edgesObservable,
- final SerializationFig serializationFig ) {
- this.emf = emf;
+ public EntityVersionDeletedHandler( final EdgesObservable edgesObservable, final SerializationFig serializationFig,
+ final EntityIndexFactory entityIndexFactory,
+ final GraphManagerFactory graphManagerFactory ) {
this.edgesObservable = edgesObservable;
this.serializationFig = serializationFig;
+ this.entityIndexFactory = entityIndexFactory;
+ this.graphManagerFactory = graphManagerFactory;
}
@@ -87,10 +92,8 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
} );
}
- CpEntityManagerFactory cpemf = ( CpEntityManagerFactory ) emf;
-
- final ApplicationEntityIndex ei = cpemf.getManagerCache().getEntityIndex( scope );
- final GraphManager gm = cpemf.getManagerCache().getGraphManager( scope );
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( scope );
+ final GraphManager gm = graphManagerFactory.createEdgeManager( scope );
//create an observable of all scopes to deIndex
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
new file mode 100644
index 0000000..fc1bdb7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+
+
+/**
+ * A temporary interface of our buffer Q to decouple of producer and consumer;
+ */
+public interface BufferQueue {
+
+ /**
+ * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
+ * Other queues may return the future after ack on the offer. See the implementation documentation for details.
+ * @param operation
+ */
+ public void offer(final IndexOperationMessage operation);
+
+
+ /**
+ * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
+ * May return less than the take size, but will never return null
+ *
+ * @param takeSize
+ * @param timeout
+ * @param timeUnit
+ * @return A null safe lid
+ */
+ public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
+
+
+ /**
+ * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
+ * This will set the future as done in in memory operations
+ *
+ * @param messages
+ */
+ public void ack(final List<IndexOperationMessage> messages);
+
+ /**
+ * Mark these message as failed. Set the exception in the future on local operation
+ *
+ * @param messages
+ */
+ public void fail(final List<IndexOperationMessage> messages, final Throwable t);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
new file mode 100644
index 0000000..0e43da3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class BufferQueueInMemoryImpl implements BufferQueue {
+
+
+ private final QueryFig fig;
+ private final ArrayBlockingQueue<IndexOperationMessage> messages;
+
+
+ @Inject
+ public BufferQueueInMemoryImpl( final QueryFig fig ) {
+ this.fig = fig;
+ messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+ try {
+ messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
+ }
+ catch ( InterruptedException e ) {
+ throw new RuntimeException("Unable to offer message to queue", e);
+ }
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
+ try {
+
+
+ messages.drainTo( response, takeSize );
+
+ //we got something, go process it
+ if ( response.size() > 0 ) {
+ return response;
+ }
+
+
+ final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
+
+ if ( polled != null ) {
+ response.add( polled );
+
+ //try to add more
+ messages.drainTo( response, takeSize - 1 );
+ }
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+
+
+ return response;
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+ //if we have a future ack it
+ for ( final IndexOperationMessage op : messages ) {
+ op.done();
+ }
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+
+
+ for ( final IndexOperationMessage op : messages ) {
+ final BetterFuture<IndexOperationMessage> future = op.getFuture();
+
+ if ( future != null ) {
+ future.setError( t );
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
new file mode 100644
index 0000000..d955014
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
+ * performing
+ */
+@Singleton
+public class BufferQueueSQSImpl implements BufferQueue {
+
+ private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final MapManager mapManager;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope =
+ new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
+
+ this.mapManager = mapManagerFactory.createMapManager( scope );
+
+
+ this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ @Override
+ public void offer( final IndexOperationMessage operation ) {
+
+ //no op
+ if(operation.isEmpty()){
+ operation.getFuture().done();
+ return;
+ }
+
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //write to cassandra
+ this.mapManager.putString( identifier.toString(), payLoad, TTL );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ operation.done();
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ @Override
+ public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+
+ final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if(messages.size() == 0){
+ return response;
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+ //look up the values
+ final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String key = getMessageKey( message );
+
+ //now see if the key was there
+ final String payload = storedCommands.get( key );
+
+ //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
+ // a DLQ
+
+ if ( payload == null ) {
+ continue;
+ }
+
+ final IndexOperationMessage messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ @Override
+ public void ack( final List<IndexOperationMessage> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexOperationMessage ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
+
+ //remove it from the map
+ mapManager.delete( key );
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ @Override
+ public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
+ //no op, just let it retry after the queue timeout
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexOperationMessage fromString( String s ) throws IOException {
+ IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexOperationMessage o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+ private String getMessageKey(final QueueMessage message){
+ return message.getBody().toString();
+ }
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexOperationMessage {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexOperationMessage source ) {
+ this.message = message;
+ this.addAllDeIndexRequest( source.getDeIndexRequests() );
+ this.addAllIndexRequest( source.getIndexRequests() );
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
new file mode 100644
index 0000000..8d7f222
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface IndexQueueService {
+
+
+ /**
+ * Queue an entity to be index asynchronously
+ * @param applicationScope
+ * @param entityId
+ * @param version
+ */
+ void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
new file mode 100644
index 0000000..2bf073c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Our low level indexing service operations
+ */
+public interface IndexService {
+
+
+ /**
+ * Perform an index update of the entity's state from Cassandra
+ *
+ * @param applicationScope The scope of the entity
+ * @param entity The entity
+ *
+ * @return An observable with the count of every
+ */
+ Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity );
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
new file mode 100644
index 0000000..2a7533a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeToTarget;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+
+
+/**
+ * Implementation of the indexing service
+ */
+@Singleton
+public class IndexServiceImpl implements IndexService {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
+ private final EdgesObservable edgesObservable;
+ private final IndexFig indexFig;
+
+
+ @Inject
+ public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory,
+ final EdgesObservable edgesObservable, IndexFig indexFig ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
+ this.edgesObservable = edgesObservable;
+ this.indexFig = indexFig;
+ }
+
+
+ @Override
+ public Observable<Integer> indexEntity( final ApplicationScope applicationScope, final Entity entity ) {
+
+
+ //bootstrap the lower modules from their caches
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+
+ final Id entityId = entity.getId();
+
+
+ //we always index in the target scope
+ final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
+
+ //we may have to index
+ final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeToTarget( edge ) );
+
+
+ //we might or might not need to index from target-> source
+
+
+ final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
+
+
+ final Observable<IndexOperationMessage> observable =
+ //try to send a whole batch if we can
+ Observable.merge( sourceEdgesToIndex, targetSizes ).buffer( indexFig.getIndexBatchSize() )
+
+ //map into batches based on our buffer size
+ .flatMap( buffer -> Observable.from( buffer ).collect( () -> ei.createBatch(),
+ ( batch, indexEdge ) -> batch.index( indexEdge, entity ) )
+ //return the future from the batch execution
+ .flatMap( batch -> Observable.from( batch.execute() ) ) );
+
+ observable.toBlocking().last();
+
+
+ return Observable.just( 0 );
+ }
+
+
+ /**
+ * Get index edgs to the target
+ *
+ * @param graphManager The graph manager
+ * @param entityId The entitie's id
+ */
+ private Observable<IndexEdge> getIndexEdgesToTarget( final GraphManager graphManager, final Id entityId ) {
+
+ final String collectionName = InflectionUtils.pluralize( entityId.getType() );
+
+
+ final CollectionInfo collection = getDefaultSchema().getCollection( Application.ENTITY_TYPE, collectionName );
+
+ //nothing to do
+ if ( collection == null ) {
+ return Observable.empty();
+ }
+
+
+ final String linkedCollection = collection.getLinkedCollection();
+
+ /**
+ * Nothing to link
+ */
+ if ( linkedCollection == null ) {
+ return Observable.empty();
+ }
+
+
+ /**
+ * An observable of sizes as we execute batches
+ */
+ return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
+ .map( edge -> generateScopeFromSource( edge ) );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
new file mode 100644
index 0000000..a7d2450
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ * Application id cache fig
+ */
+public interface QueryFig extends GuicyFig {
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+
+ /**
+ * The number of worker threads to consume from the queue
+ */
+ public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
+
+ /**
+ * Amount of time to wait when reading from the queue
+ */
+ public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
+
+ /**
+ * Amount of time to wait when reading from the queue in milliseconds
+ */
+ public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
+
+
+ String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
+
+
+ @Default( "1000" )
+ @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
+ long getFailureRetryTime();
+
+ //give us 60 seconds to process the message
+ @Default( "60" )
+ @Key( INDEX_QUEUE_READ_TIMEOUT )
+ int getIndexQueueTimeout();
+
+ @Default( "2" )
+ @Key( ELASTICSEARCH_WORKER_COUNT )
+ int getWorkerCount();
+
+ @Default( "LOCAL" )
+ @Key( ELASTICSEARCH_QUEUE_IMPL )
+ String getQueueImplementation();
+
+ @Default( "1000" )
+ @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
+ long getQueueOfferTimeout();
+
+ /**
+ * size of the buffer to build up before you send results
+ */
+ @Default( "1000" )
+ @Key( INDEX_QUEUE_SIZE )
+ int getIndexQueueSize();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
new file mode 100644
index 0000000..d3920db
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class QueueProvider implements Provider<BufferQueue> {
+
+ private final QueryFig queryFig;
+
+ private final QueueManagerFactory queueManagerFactory;
+ private final MapManagerFactory mapManagerFactory;
+ private final MetricsFactory metricsFactory;
+
+ private BufferQueue bufferQueue;
+
+
+ @Inject
+ public QueueProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
+ final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
+ this.queryFig = queryFig;
+
+
+ this.queueManagerFactory = queueManagerFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.metricsFactory = metricsFactory;
+ }
+
+
+ @Override
+ @Singleton
+ public BufferQueue get() {
+ if ( bufferQueue == null ) {
+ bufferQueue = getQueue();
+ }
+
+
+ return bufferQueue;
+ }
+
+
+ private BufferQueue getQueue() {
+ final String value = queryFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new BufferQueueInMemoryImpl( queryFig );
+ case SQS:
+ return new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
new file mode 100644
index 0000000..42f36b1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class SQSIndexQueueServiceImpl implements IndexQueueService {
+
+ @Override
+ public void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version ) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 0376780..c60b86e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -157,7 +157,7 @@ public class CpNamingUtils {
*/
public static SearchEdge createCollectionSearchEdge( final Id sourceId, final String connectionType ) {
return new SearchEdgeImpl( sourceId, getEdgeTypeFromCollectionName( connectionType ),
- SearchEdge.NodeType.SOURCE );
+ SearchEdge.NodeType.TARGET );
}
@@ -181,7 +181,7 @@ public class CpNamingUtils {
*/
public static SearchEdge createConnectionSearchEdge( final Id sourceId, final String connectionType ) {
return new SearchEdgeImpl( sourceId, getEdgeTypeFromConnectionType( connectionType ),
- SearchEdge.NodeType.SOURCE );
+ SearchEdge.NodeType.TARGET );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 7929b12..fc8b3d5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -689,8 +689,6 @@ public interface EntityManager {
/** @return the cass */
CassandraService getCass();
- public void init( EntityManagerFactory emf, UUID applicationId);
-
/** For testing purposes */
public void flushManagerCaches();
[02/52] [abbrv] incubator-usergrid git commit: Initial pass at moving
queues to core
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
new file mode 100644
index 0000000..8bda3b0
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/GuiceAdapterBeanFactory.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence;
+
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.support.DefaultListableBeanFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+import com.google.inject.Injector;
+
+
+/**
+ * This allows us to search guice for beans that are not in spring
+ */
+@Component
+public class GuiceAdapterBeanFactory extends DefaultListableBeanFactory {
+
+ /**
+ * Wire our injector into this so we can use it go get beans
+ */
+ @Autowired
+ private Injector injector;
+
+
+ public <T> T getBean( Class<T> requiredType ) throws BeansException {
+ final T bean = super.getBean( requiredType );
+
+ // Comes from spring, return it
+ if ( bean != null ) {
+ return bean;
+ }
+
+ final T guiceBean = injector.getInstance( requiredType );
+
+ if(guiceBean == null){
+ throw new NoGuiceBean( "Could not find bean for class" + requiredType );
+ }
+
+
+ return guiceBean;
+ }
+
+
+ @Override
+ public Object getBean( final String name ) throws BeansException {
+ final Object springBean = super.getBean( name );
+
+ return validateBean( springBean, name );
+ }
+
+
+ @Override
+ public <T> T getBean( final String name, final Class<T> requiredType ) throws BeansException {
+ final T springBean = super.getBean( name, requiredType );
+
+ return validateBean( springBean, name );
+ }
+
+
+ @Override
+ public <T> T getBean( final String name, final Class<T> requiredType, final Object... args ) throws BeansException {
+ final T springBean = super.getBean( name, requiredType, args );
+
+ return validateBean( springBean, name );
+ }
+
+
+ /**
+ * If we can't find the spring bean, we should blow up
+ * @param springBean
+ * @param <T>
+ * @return
+ */
+ private <T> T validateBean( T springBean, final String name ) {
+ if ( springBean == null ) {
+ throw new NoGuiceBean( String.format("Guice beans by name is unsupoported, and could not find a spring bean with the name '%s'", name) );
+ }
+
+ return springBean;
+ }
+
+
+ /**
+ * Exception class to throw when we can't find a bean in spring, and can't find it in guice
+ */
+ public static final class NoGuiceBean extends BeansException{
+
+ public NoGuiceBean( final String msg ) {
+ super( msg );
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
new file mode 100644
index 0000000..6e58676
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PersistenceModule.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence;
+
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.Properties;
+
+import org.springframework.context.ApplicationContext;
+
+import org.apache.usergrid.locking.cassandra.HectorLockManagerImpl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.CharSource;
+import com.google.common.io.Resources;
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
+import com.google.inject.spring.SpringIntegration;
+
+import me.prettyprint.cassandra.connection.RoundRobinBalancingPolicy;
+import me.prettyprint.cassandra.service.CassandraHostConfigurator;
+import me.prettyprint.cassandra.service.ThriftCluster;
+import me.prettyprint.hector.api.Cluster;
+
+
+/**
+ * Replacement for configuration of our spring modules with guice
+ */
+public class PersistenceModule extends AbstractModule {
+
+
+
+ private final ApplicationContext applicationContext;
+
+ public PersistenceModule( final ApplicationContext applicationContext ) {
+
+ this.applicationContext = applicationContext;
+ }
+
+
+
+
+ @Override
+ protected void configure() {
+ SpringIntegration.bindAll( binder(), applicationContext );
+ }
+
+
+
+// <bean id="cassandraCluster" class="me.prettyprint.cassandra.service.ThriftCluster">
+// <constructor-arg value="${cassandra.cluster}" />
+// <constructor-arg ref="cassandraHostConfigurator" />
+// </bean>
+// @Provides
+// @Singleton
+// @Inject
+// public Cluster configureThrift( @Named( "cassandra.cluster" ) final String cassCluster,
+// @Named( "cassandra.connections" ) final int cassandraConnections ){
+//
+// final int setSize = cassandraConnections == 0 ? 50: cassandraConnections;
+//
+// CassandraHostConfigurator hostConfigurator = new CassandraHostConfigurator( cassCluster );
+//
+// hostConfigurator.setMaxActive( setSize );
+// hostConfigurator.setLoadBalancingPolicy( new RoundRobinBalancingPolicy() );
+//
+//
+// ThriftCluster thriftCluster = new ThriftCluster(cassCluster, hostConfigurator);
+//
+// return thriftCluster;
+//
+// }
+//
+//
+// @Provides
+// @Singleton
+// @Inject
+// public Properties configureProps(final PropertiesProvider propertiesProvider ){
+//
+// final Properties props = new Properties( );
+//
+// for(final String propFile: propertiesProvider.getPropertiesFiles()){
+//
+// final URL url = Resources.getResource( propFile );
+//
+// Preconditions.checkNotNull( url, "Could not find properties file '" + propFile + "' on the classpath" );
+//
+//
+// final CharSource propsInput = Resources.asCharSource( url, Charset.defaultCharset() );
+// try {
+// props.load( propsInput.openStream() );
+// }
+// catch ( IOException e ) {
+// throw new RuntimeException( "Unable to load properties file '" + propFile + "'", e );
+// }
+// }
+//
+// //bind these properties
+// Names.bindProperties( binder(), props );
+//
+// return props;
+// }
+//
+// @Provides
+// @Singleton
+// @Inject
+// public void configureLocks(final Cluster hectorCluster, @Named("cassandra.lock.keyspace") final String lockKeyspace, @Named("cassandra.lock.keyspace") final String writeCl, final String readCl ){
+//
+//
+// final HectorLockManagerImpl hectorLockManager = new HectorLockManagerImpl();
+//
+//
+////
+//// <bean name="consistencyLevelPolicy" class="me.prettyprint.cassandra.model.ConfigurableConsistencyLevel">
+//// <property name="defaultReadConsistencyLevel" value="${cassandra.readcl}"/>
+//// <property name="defaultWriteConsistencyLevel" value="${cassandra.writecl}"/>
+//// </bean>
+//
+//// <bean name="lockManager" class="org.apache.usergrid.locking.cassandra.HectorLockManagerImpl" >
+//// <property name="cluster" ref="cassandraCluster"/>
+//// <property name="keyspaceName" value="${cassandra.lock.keyspace}"/>
+//// <property name="consistencyLevelPolicy" ref="consistencyLevelPolicy"/>
+//// </bean>
+//
+// }
+//
+//
+// /**
+// * Interface to allow users to provide and inject properties
+// */
+// public interface PropertiesProvider{
+// /**
+// * Get the properties files to load
+// * @return
+// */
+// String[] getPropertiesFiles();
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
new file mode 100644
index 0000000..e5254d1
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestIndexModule.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence;
+
+
+import org.apache.usergrid.persistence.core.guice.TestModule;
+
+
+public class TestIndexModule extends TestModule {
+
+ @Override
+ protected void configure() {
+
+ //this will break, we need to untagle this and move to guice in core completely
+ install( new CoreModule( ));
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
new file mode 100644
index 0000000..a76a589
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+@NotThreadSafe
+public class BufferQueueSQSImplTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+ @Inject
+ public QueueManagerFactory queueManagerFactory;
+
+ @Inject
+ public QueryFig queryFig;
+
+ @Inject
+ public MapManagerFactory mapManagerFactory;
+
+ @Inject
+ public MetricsFactory metricsFactory;
+
+
+ private BufferQueueSQSImpl bufferQueueSQS;
+
+ @Before
+ public void setup(){
+ bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
+ }
+
+
+
+
+ @Test
+ public void testMessageIndexing(){
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+ assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
+ assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
+
+ final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
+ final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
+
+
+ final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
+ final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
+
+
+ //de-index request
+ final DeIndexRequest
+ deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
+
+
+ SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
+
+ final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
+ SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
+
+
+
+
+ IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
+ indexOperationMessage.addIndexRequest( indexRequest1);
+ indexOperationMessage.addIndexRequest( indexRequest2);
+
+ indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
+ indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
+
+ bufferQueueSQS.offer( indexOperationMessage );
+
+ //wait for it to send to SQS
+ indexOperationMessage.getFuture().get();
+
+ //now get it back
+
+ final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
+
+ assertTrue(ops.size() > 0);
+
+ final IndexOperationMessage returnedOperation = ops.get( 0 );
+
+ //get the operations out
+
+ final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
+
+ assertTrue(indexRequestSet.contains(indexRequest1));
+ assertTrue(indexRequestSet.contains(indexRequest2));
+
+
+ final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
+
+ assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
+ assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
+
+
+
+ //now ack the message
+
+ bufferQueueSQS.ack( ops );
+
+ }
+
+ private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
+ final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+
+ List<IndexOperationMessage> ops;
+
+ do{
+ ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+ }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
+
+ return ops;
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
new file mode 100644
index 0000000..1b0f538
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/aws/NoAWSCredsRule.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.aws;
+
+
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import com.amazonaws.AmazonClientException;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule implements TestRule {
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+ try {
+ base.evaluate();
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ if ( t instanceof AmazonClientException ) {
+
+ final AmazonClientException ace = ( AmazonClientException ) t;
+
+ if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+ "could not get aws secret key from system properties" ) ) {
+ //swallow
+ return true;
+ }
+ }
+
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 2206953..fa73991 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -28,6 +28,29 @@ import rx.Observable;
* Get all edges from source
*/
public interface EdgesObservable {
+
+ /**
+ * Return an observable of all edges from a source
+ * @param gm
+ * @param sourceNode
+ * @return
+ */
Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode);
+
+ /**
+ * Get all edges from the source node with the target type
+ * @param gm
+ * @param sourceNode
+ * @param targetType
+ * @return
+ */
+ Observable<Edge> getEdgesFromSource(final GraphManager gm, final Id sourceNode, final String targetType );
+
+ /**
+ * Return an observable of all edges to a target
+ * @param gm
+ * @param targetNode
+ * @return
+ */
Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 371cf1d..2264cbd 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -19,62 +19,88 @@
*/
package org.apache.usergrid.persistence.graph.serialization.impl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
import rx.Observable;
import rx.functions.Func1;
+
/**
* Emits the edges that are edges from the specified source node
*/
public class EdgesObservableImpl implements EdgesObservable {
- private static final Logger logger = LoggerFactory.getLogger(EdgesObservableImpl.class);
- public EdgesObservableImpl(){
+ private static final Logger logger = LoggerFactory.getLogger( EdgesObservableImpl.class );
+
+
+ public EdgesObservableImpl() {
}
+
/**
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode){
- Observable<String> edgeTypes = gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+ public Observable<Edge> edgesFromSource( final GraphManager gm, final Id sourceNode ) {
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
- return edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
+ return edgeTypes.flatMap( edgeType -> {
logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
- return gm.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
- }
+ return gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ null ) );
} );
}
+
+
+ @Override
+ public Observable<Edge> getEdgesFromSource( final GraphManager gm, final Id sourceNode, final String targetType ) {
+
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
+
+
+ return edgeTypes.flatMap( edgeType -> {
+
+ logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
+
+ return gm.loadEdgesFromSourceByType(
+ new SimpleSearchByIdType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ targetType, null ) );
+ } );
+ }
+
+
/**
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode) {
- Observable<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
+ public Observable<Edge> edgesToTarget( final GraphManager gm, final Id targetNode ) {
+ final Observable<String> edgeTypes =
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
- return edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
+ return edgeTypes.flatMap( edgeType -> {
- logger.debug( "Loading edges of edgeType {} to {}", edgeType, targetNode);
+ logger.debug( "Loading edges of edgeType {} to {}", edgeType, targetNode );
- return gm.loadEdgesToTarget( new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
- }
+ return gm.loadEdgesToTarget(
+ new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ null ) );
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index da05d39..39c686c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index;/*
import java.util.UUID;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -65,7 +66,7 @@ public interface EntityIndexBatch {
* Execute the batch
* @return future to guarantee execution
*/
- BetterFuture execute();
+ BetterFuture<IndexOperationMessage> execute();
/**
* Get the number of operations in the batch
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 8b9a03e..d13d055 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -30,84 +30,52 @@ import org.apache.usergrid.persistence.index.impl.EsProvider;
@FigSingleton
public interface IndexFig extends GuicyFig {
- public static final String ELASTICSEARCH_HOSTS = "elasticsearch.hosts";
+ String ELASTICSEARCH_HOSTS = "elasticsearch.hosts";
- public static final String ELASTICSEARCH_PORT = "elasticsearch.port";
+ String ELASTICSEARCH_PORT = "elasticsearch.port";
- public static final String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster_name";
+ String ELASTICSEARCH_CLUSTER_NAME = "elasticsearch.cluster_name";
- public static final String ELASTICSEARCH_NODENAME = "elasticsearch.node_name";
+ String ELASTICSEARCH_NODENAME = "elasticsearch.node_name";
- public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
+ String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
- public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch.alias_postfix";
+ String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch.alias_postfix";
- public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
+ String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
- public static final String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
+ String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
- public static final String ELASTICSEARCH_NUMBER_OF_REPLICAS = "elasticsearch.number_replicas";
+ String ELASTICSEARCH_NUMBER_OF_REPLICAS = "elasticsearch.number_replicas";
- public static final String QUERY_CURSOR_TIMEOUT_MINUTES = "elasticsearch.cursor_timeout.minutes";
+ String QUERY_CURSOR_TIMEOUT_MINUTES = "elasticsearch.cursor_timeout.minutes";
- public static final String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
+ String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
- public static final String INDEX_BUFFER_SIZE = "elasticsearch.buffer_size";
+ String INDEX_BUFFER_SIZE = "elasticsearch.buffer_size";
- public static final String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
+ String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
- public static final String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
+ String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
- /**
- * Amount of time to wait when reading from the queue
- */
- public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
-
- /**
- * Amount of time to wait when reading from the queue in milliseconds
- */
- public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
-
- public static final String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
-
- public static final String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
+ String INDEX_WRITE_CONSISTENCY_LEVEL = "elasticsearch.write_consistency_level";
/**
* the number of times we can fail before we refresh the client
*/
- public static final String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
-
- /**
- * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
- * backpressure
- */
- public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+ String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
- /**
- * The number of worker threads to consume from the queue
- */
- public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
-
-
- public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
/**
* The client type to use. Valid values are NODE or TRANSPORT
*/
- public static final String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+ String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
@Default( "127.0.0.1" )
@@ -145,20 +113,20 @@ public interface IndexFig extends GuicyFig {
@Default( "false" )
@Key( ELASTICSEARCH_FORCE_REFRESH )
- public boolean isForcedRefresh();
+ boolean isForcedRefresh();
/** Identify the client node with a unique name. */
@Default( "default" )
@Key( ELASTICSEARCH_NODENAME )
- public String getNodeName();
+ String getNodeName();
@Default( "6" )
@Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
- public int getNumberOfShards();
+ int getNumberOfShards();
@Default( "1" )
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
- public int getNumberOfReplicas();
+ int getNumberOfReplicas();
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
@@ -181,12 +149,7 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_BUFFER_SIZE )
int getIndexBufferSize();
- /**
- * size of the buffer to build up before you send results
- */
- @Default( "1000" )
- @Key( INDEX_QUEUE_SIZE )
- int getIndexQueueSize();
+
/**
* Request batch size for ES
@@ -199,27 +162,6 @@ public interface IndexFig extends GuicyFig {
@Key( INDEX_WRITE_CONSISTENCY_LEVEL )
String getWriteConsistencyLevel();
- @Default( "1000" )
- @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
- long getFailureRetryTime();
-
- //give us 60 seconds to process the message
- @Default( "60" )
- @Key( INDEX_QUEUE_READ_TIMEOUT )
- int getIndexQueueTimeout();
-
- @Default( "2" )
- @Key( ELASTICSEARCH_WORKER_COUNT )
- int getWorkerCount();
-
- @Default( "LOCAL" )
- @Key( ELASTICSEARCH_QUEUE_IMPL )
- String getQueueImplementation();
-
- @Default( "1000" )
- @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
- long getQueueOfferTimeout();
-
/**
* Return the type of client. Valid values or NODE or TRANSPORT
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 10cbde0..b6a1f09 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -52,15 +52,12 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndex.class).to(EsEntityIndexImpl.class).asEagerSingleton();
bind(IndexCache.class).to(EsIndexCacheImpl.class);
bind(IndexRefreshCommand.class).to(IndexRefreshCommandImpl.class);
- bind(FailureMonitorImpl.IndexIdentifier.class).to(IndexIdentifierImpl.class);
+ bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
- bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
- bind( BufferQueue.class).toProvider( QueueProvider.class );
-
//wire up the edg migration. A no-op ATM, but retained for future development
Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder =
Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
deleted file mode 100644
index ea3e046..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/QueueProvider.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.guice;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
-import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-
-
-/**
- * A provider to allow users to configure their queue impl via properties
- */
-@Singleton
-public class QueueProvider implements Provider<BufferQueue> {
-
- private final IndexFig indexFig;
-
- private final QueueManagerFactory queueManagerFactory;
- private final MapManagerFactory mapManagerFactory;
- private final MetricsFactory metricsFactory;
-
- private BufferQueue bufferQueue;
-
-
- @Inject
- public QueueProvider( final IndexFig indexFig, final QueueManagerFactory queueManagerFactory,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- this.indexFig = indexFig;
-
-
- this.queueManagerFactory = queueManagerFactory;
- this.mapManagerFactory = mapManagerFactory;
- this.metricsFactory = metricsFactory;
- }
-
-
- @Override
- @Singleton
- public BufferQueue get() {
- if ( bufferQueue == null ) {
- bufferQueue = getQueue();
- }
-
-
- return bufferQueue;
- }
-
-
- private BufferQueue getQueue() {
- final String value = indexFig.getQueueImplementation();
-
- final Implementations impl = Implementations.valueOf( value );
-
- switch ( impl ) {
- case LOCAL:
- return new BufferQueueInMemoryImpl( indexFig );
- case SQS:
- return new BufferQueueSQSImpl( queueManagerFactory, indexFig, mapManagerFactory, metricsFactory );
- default:
- throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
- }
- }
-
-
- private String getErrorValues() {
- String values = "";
-
- for ( final Implementations impl : Implementations.values() ) {
- values += impl + ", ";
- }
-
- values = values.substring( 0, values.length() - 2 );
-
- return values;
- }
-
-
- /**
- * Different implementations
- */
- public static enum Implementations {
- LOCAL,
- SQS;
-
-
- public String asString() {
- return toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
deleted file mode 100644
index cb89f64..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueue.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-
-/**
- * A temporary interface of our buffer Q to decouple of producer and consumer;
- */
-public interface BufferQueue {
-
- /**
- * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
- * Other queues may return the future after ack on the offer. See the implementation documentation for details.
- * @param operation
- */
- public void offer(final IndexIdentifierImpl.IndexOperationMessage operation);
-
-
- /**
- * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
- * May return less than the take size, but will never return null
- *
- * @param takeSize
- * @param timeout
- * @param timeUnit
- * @return A null safe lid
- */
- public List<IndexIdentifierImpl.IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
-
-
- /**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
- * This will set the future as done in in memory operations
- *
- * @param messages
- */
- public void ack(final List<IndexIdentifierImpl.IndexOperationMessage> messages);
-
- /**
- * Mark these message as failed. Set the exception in the future on local operation
- *
- * @param messages
- */
- public void fail(final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
deleted file mode 100644
index bfaed3d..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.index.IndexFig;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class BufferQueueInMemoryImpl implements BufferQueue {
-
-
- private final IndexFig fig;
- private final ArrayBlockingQueue<IndexIdentifierImpl.IndexOperationMessage> messages;
-
-
- @Inject
- public BufferQueueInMemoryImpl( final IndexFig fig ) {
- this.fig = fig;
- messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
- }
-
-
- @Override
- public void offer( final IndexIdentifierImpl.IndexOperationMessage operation ) {
- try {
- messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
- }
- catch ( InterruptedException e ) {
- throw new RuntimeException("Unable to offer message to queue", e);
- }
- }
-
-
- @Override
- public List<IndexIdentifierImpl.IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- final List<IndexIdentifierImpl.IndexOperationMessage> response = new ArrayList<>( takeSize );
- try {
-
-
- messages.drainTo( response, takeSize );
-
- //we got something, go process it
- if ( response.size() > 0 ) {
- return response;
- }
-
-
- final IndexIdentifierImpl.IndexOperationMessage polled = messages.poll( timeout, timeUnit );
-
- if ( polled != null ) {
- response.add( polled );
-
- //try to add more
- messages.drainTo( response, takeSize - 1 );
- }
- }
- catch ( InterruptedException e ) {
- //swallow
- }
-
-
- return response;
- }
-
-
- @Override
- public void ack( final List<IndexIdentifierImpl.IndexOperationMessage> messages ) {
- //if we have a future ack it
- for ( final IndexIdentifierImpl.IndexOperationMessage op : messages ) {
- op.done();
- }
- }
-
-
- @Override
- public void fail( final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t ) {
-
-
- for ( final IndexIdentifierImpl.IndexOperationMessage op : messages ) {
- final BetterFuture<IndexIdentifierImpl.IndexOperationMessage> future = op.getFuture();
-
- if ( future != null ) {
- future.setError( t );
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
deleted file mode 100644
index fee828f..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
- * performing
- */
-@Singleton
-public class BufferQueueSQSImpl implements BufferQueue {
-
- private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final MapManager mapManager;
- private final IndexFig indexFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final IndexFig indexFig,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- final QueueScope queueScope =
- new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.indexFig = indexFig;
-
- final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
-
- this.mapManager = mapManagerFactory.createMapManager( scope );
-
-
- this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- @Override
- public void offer( final IndexIdentifierImpl.IndexOperationMessage operation ) {
-
- //no op
- if(operation.isEmpty()){
- operation.getFuture().done();
- return;
- }
-
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //write to cassandra
- this.mapManager.putString( identifier.toString(), payLoad, TTL );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- operation.done();
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public List<IndexIdentifierImpl.IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, indexFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
-
- final List<IndexIdentifierImpl.IndexOperationMessage> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if(messages.size() == 0){
- return response;
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
- //look up the values
- final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String key = getMessageKey( message );
-
- //now see if the key was there
- final String payload = storedCommands.get( key );
-
- //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
- // a DLQ
-
- if ( payload == null ) {
- continue;
- }
-
- final IndexIdentifierImpl.IndexOperationMessage messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public void ack( final List<IndexIdentifierImpl.IndexOperationMessage> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexIdentifierImpl.IndexOperationMessage ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
-
- //remove it from the map
- mapManager.delete( key );
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- @Override
- public void fail( final List<IndexIdentifierImpl.IndexOperationMessage> messages, final Throwable t ) {
- //no op, just let it retry after the queue timeout
- }
-
-
- /** Read the object from Base64 string. */
- private IndexIdentifierImpl.IndexOperationMessage fromString( String s ) throws IOException {
- IndexIdentifierImpl.IndexOperationMessage o = mapper.readValue( s, IndexIdentifierImpl.IndexOperationMessage.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexIdentifierImpl.IndexOperationMessage o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
- private String getMessageKey(final QueueMessage message){
- return message.getBody().toString();
- }
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexIdentifierImpl.IndexOperationMessage {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexIdentifierImpl.IndexOperationMessage source ) {
- this.message = message;
- this.addAllDeIndexRequest( source.getDeIndexRequests() );
- this.addAllIndexRequest( source.getIndexRequests() );
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index b509ad9..05c84b9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.elasticsearch.action.ActionListener;
@@ -61,14 +60,11 @@ import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import rx.Observable;
@@ -85,12 +81,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final ApplicationScope applicationScope;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
private final Timer searchTimer;
private final Timer cursorTimer;
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -103,11 +99,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
@Inject
public EsApplicationEntityIndexImpl( ApplicationScope appScope, final AliasedEntityIndex entityIndex,
- final IndexFig config, final IndexBufferProducer indexBatchBufferProducer,
+ final IndexFig config, final IndexBufferConsumer indexBatchBufferProducer,
final EsProvider provider,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig,
- final FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
+ final IndexIdentifier indexIdentifier ) {
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.indexFig = indexFig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index d22f000..32a0f02 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -44,18 +44,18 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final ApplicationScope applicationScope;
private final IndexAlias alias;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final AliasedEntityIndex entityIndex;
- private IndexIdentifierImpl.IndexOperationMessage container;
+ private IndexOperationMessage container;
public EsEntityIndexBatchImpl( final ApplicationScope applicationScope,
- final IndexBufferProducer indexBatchBufferProducer,
+ final IndexBufferConsumer indexBatchBufferProducer,
final AliasedEntityIndex entityIndex,
- FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
+ IndexIdentifier indexIdentifier ) {
this.applicationScope = applicationScope;
this.indexBatchBufferProducer = indexBatchBufferProducer;
@@ -63,7 +63,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.indexIdentifier = indexIdentifier;
this.alias = indexIdentifier.getAlias();
//constrained
- this.container = new IndexIdentifierImpl.IndexOperationMessage();
+ this.container = new IndexOperationMessage();
}
@@ -126,8 +126,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public BetterFuture execute() {
- IndexIdentifierImpl.IndexOperationMessage tempContainer = container;
- container = new IndexIdentifierImpl.IndexOperationMessage();
+ IndexOperationMessage tempContainer = container;
+ container = new IndexOperationMessage();
/**
* No-op, just disregard it
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 6145069..e41bcf8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -42,12 +42,12 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexBufferConsumer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
private final IndexFig indexFig;
private final AliasedEntityIndex entityIndex;
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+ private final IndexIdentifier indexIdentifier;
private LoadingCache<ApplicationScope, ApplicationEntityIndex> eiCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, ApplicationEntityIndex>() {
@@ -60,9 +60,9 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
@Inject
public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider,
- final IndexBufferProducer indexBatchBufferProducer,
+ final IndexBufferConsumer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
- final IndexFig indexFig, final AliasedEntityIndex entityIndex, final FailureMonitorImpl.IndexIdentifier indexIdentifier ){
+ final IndexFig indexFig, final AliasedEntityIndex entityIndex, final IndexIdentifier indexIdentifier ){
this.config = config;
this.provider = provider;
this.indexBatchBufferProducer = indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/84d779fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 456bd15..961ddd2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -25,57 +25,33 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.sun.org.apache.xpath.internal.operations.Bool;
-import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
-import org.apache.usergrid.persistence.index.query.ParsedQuery;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.EntityUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
-import org.elasticsearch.action.index.*;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
-import org.elasticsearch.indices.IndexMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import rx.*;
+
import rx.Observable;
-import rx.functions.Action1;
import java.io.IOException;
import java.net.URL;
import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -87,8 +63,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final IndexAlias alias;
- private final IndexBufferProducer indexBatchBufferProducer;
- private final MetricsFactory metricsFactory;
+ private final IndexBufferConsumer producer;
private final IndexFig indexFig;
private final Timer addTimer;
private final Timer updateAliasTimer;
@@ -97,7 +72,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
* We purposefully make this per instance. Some indexes may work, while others may fail
*/
private final EsProvider esProvider;
- private final IndexBufferProducer producer;
private final IndexRefreshCommand indexRefreshCommand;
//number of times to wait for the index to refresh properly.
@@ -110,8 +84,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
ImmutableMap.<String, Object>builder().put(IndexingUtils.ENTITY_ID_FIELDNAME, UUIDGenerator.newTimeUUID().toString()).build();
- private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
- private final FailureMonitorImpl.IndexIdentifier indexIdentifier;
+
+ private final IndexIdentifier indexIdentifier;
private IndexCache aliasCache;
private Timer mappingTimer;
@@ -121,12 +95,12 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
@Inject
- public EsEntityIndexImpl(final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final IndexCache indexCache, final MetricsFactory metricsFactory,
- final IndexFig indexFig, final FailureMonitorImpl.IndexIdentifier indexIdentifier,
- final IndexBufferProducer producer, IndexRefreshCommand indexRefreshCommand) {
- this.indexBatchBufferProducer = indexBatchBufferProducer;
- this.metricsFactory = metricsFactory;
+ public EsEntityIndexImpl( final EsProvider provider,
+ final IndexCache indexCache,
+ final IndexFig indexFig, final IndexIdentifier indexIdentifier,
+ final IndexBufferConsumer producer, final IndexRefreshCommand indexRefreshCommand,
+ final MetricsFactory metricsFactory) {
+
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
@@ -349,7 +323,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
public Observable<Boolean> refreshAsync() {
refreshIndexMeter.mark();
- BetterFuture future = indexBatchBufferProducer.put(new IndexIdentifierImpl.IndexOperationMessage());
+ BetterFuture future = producer.put(new IndexOperationMessage());
future.get();
return indexRefreshCommand.execute();
}
[40/52] [abbrv] incubator-usergrid git commit: clear cursor
Posted by to...@apache.org.
clear cursor
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1674dc07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1674dc07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1674dc07
Branch: refs/heads/USERGRID-593
Commit: 1674dc07c9293cacfadfef81dd110fd8fdc2b5f8
Parents: 1fc18b5
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 15:25:04 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 15:25:04 2015 -0600
----------------------------------------------------------------------
.../results/ElasticSearchQueryExecutor.java | 2 ++
.../org/apache/usergrid/persistence/Query.java | 18 ++++++++++--------
2 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1674dc07/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index 98a8dac..895b457 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -160,6 +160,8 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//set offset into query
if(results.getOffset().isPresent()) {
query.setOffset(results.getOffset().get());
+ }else{
+ query.clearOffset();
}
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1674dc07/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 4273334..e5fbac9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -57,8 +57,6 @@ public class Query {
private static final Logger logger = LoggerFactory.getLogger( Query.class );
private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
-
-
public enum Level {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
@@ -324,7 +322,7 @@ public class Query {
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID( uuid ) );
+ q.addIdentifier( Identifier.fromUUID(uuid) );
return q;
}
@@ -529,14 +527,18 @@ public class Query {
public void setOffsetFromCursor(String cursor) {
if(cursor == null || cursor.length() == 0){
- return;
+ clearOffset();
+ }else {
+ byte[] bytes = Base64.decodeBase64(cursor);
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
+ setOffset(number);
}
- byte[] bytes = Base64.decodeBase64(cursor);
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
- setOffset(number);
}
+ public void clearOffset() {
+ this.offset = Optional.absent();
+ }
public void setOffset( int offset ) {
this.offset = Optional.of(offset);
[19/52] [abbrv] incubator-usergrid git commit: scroll changes
Posted by to...@apache.org.
scroll changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bbbe9631
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bbbe9631
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bbbe9631
Branch: refs/heads/USERGRID-593
Commit: bbbe96316c8d5b9c60bd85c8ebef23dcac5412f7
Parents: 8d965b3
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Apr 20 17:27:37 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Apr 20 17:27:37 2015 -0600
----------------------------------------------------------------------
.../persistence/model/entity/SimpleId.java | 15 ++-
.../index/ApplicationEntityIndex.java | 3 +-
.../persistence/index/CandidateResults.java | 8 +-
.../usergrid/persistence/index/SearchTypes.java | 4 +
.../impl/EsApplicationEntityIndexImpl.java | 120 ++++++++++---------
.../impl/SearchRequestBuilderStrategy.java | 9 +-
.../persistence/index/impl/EntityIndexTest.java | 25 ++--
7 files changed, 99 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
index 7ad4ab4..da0380b 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
@@ -33,8 +33,8 @@ import com.google.common.base.Preconditions;
public class SimpleId implements Id, Serializable {
- private final UUID uuid;
- private final String type;
+ private UUID uuid;
+ private String type;
public SimpleId( final UUID uuid, final String type ) {
Preconditions.checkNotNull( uuid, "uuid is required" );
@@ -53,18 +53,29 @@ public class SimpleId implements Id, Serializable {
this(UUIDGenerator.newTimeUUID(), type);
}
+ public SimpleId( ){
+
+ }
+
+
@Override
public UUID getUuid() {
return uuid;
}
+ public void setUuid(UUID val) {
+ uuid = val;
+ }
@Override
public String getType() {
return type;
}
+ public String setType(String val) {
+ return type = val;
+ }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 81b3028..b06cefe 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -44,7 +44,8 @@ public interface ApplicationEntityIndex {
CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
final int limit );
-
+ CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
+ final int limit, final int from );
/**
* Get next page of results from a previous cursor. Note that limit used here should be the same limit as the initial
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 14130bc..5dc5502 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -49,16 +49,13 @@ public class CandidateResults implements Iterable<CandidateResult> {
private final List<CandidateResult> candidates;
private final Collection<SelectFieldMapping> getFieldMappings;
- private final String esCursor;
public CandidateResults( List<CandidateResult> candidates,
- final Collection<SelectFieldMapping> getFieldMappings,
- final String esCursor
+ final Collection<SelectFieldMapping> getFieldMappings
) {
this.candidates = candidates;
this.getFieldMappings = getFieldMappings;
- this.esCursor = esCursor;
}
@@ -113,7 +110,4 @@ public class CandidateResults implements Iterable<CandidateResult> {
return candidates.iterator();
}
- public String getEsCursor() {
- return esCursor;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 6ba0756..0fce4c8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -53,6 +53,10 @@ public class SearchTypes {
return typeNames;
}
+ public String[] getTypes(){
+ return types;
+ }
+
/**
* Create a search that will search on the specified types
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 5b05b27..6b1f7d5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -26,6 +26,10 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
@@ -39,6 +43,7 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
+import org.mortbay.util.ajax.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -141,10 +146,14 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
return batch;
}
-
@Override
public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
final int limit ) {
+ return search(searchEdge,searchTypes,query,limit,0);
+ }
+
+ public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes, final String query,
+ final int limit, final int from ) {
IndexValidationUtils.validateSearchEdge( searchEdge );
Preconditions.checkNotNull( searchTypes, "searchTypes cannot be null" );
@@ -156,7 +165,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final ParsedQuery parsedQuery = ParsedQueryBuilder.build( query );
- final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit );
+ final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, from );
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n nodeId: {}, edgeType: {}, \n type: {}\n query: {} ",
@@ -177,14 +186,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
}
failureMonitor.success();
- return parseResults( searchResponse, parsedQuery, limit );
+ return parseResults( searchResponse, searchEdge, searchTypes, parsedQuery, limit, from );
}
public CandidateResults getNextPage( final String cursor ) {
- Preconditions.checkNotNull( cursor, "cursor is a required argument" );
-
- SearchResponse searchResponse;
+ Preconditions.checkNotNull(cursor, "cursor is a required argument");
String userCursorString = cursor;
@@ -204,37 +211,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
throw new QueryException( String.format("Could not find a cursor for the value '%s' ", userCursorString ));
}
-
-
//parse the query state
final QueryState queryState = QueryState.fromSerialized( queryStateString );
-
- //parse the query so we can get select terms
- final ParsedQuery parsedQuery = ParsedQueryBuilder.build( queryState.ql);
-
-
- logger.debug( "Executing query with cursor: {} ", queryState.esCursor );
-
-
- SearchScrollRequestBuilder ssrb =
- esProvider.getClient().prepareSearchScroll( queryState.esCursor ).setScroll( cursorTimeout + "m" );
-
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet();
- timeSearchCursor.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
-
-
- failureMonitor.success();
-
- return parseResults( searchResponse, parsedQuery, queryState.limit );
+ return search(queryState.searchEdge,queryState.searchTypes,queryState.ql,queryState.limit,queryState.from);
}
@@ -292,8 +271,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
/**
* Parse the results and return the canddiate results
*/
- private CandidateResults parseResults( final SearchResponse searchResponse, final ParsedQuery query,
- final int limit ) {
+ private CandidateResults parseResults( final SearchResponse searchResponse, final SearchEdge searchEdge,
+ final SearchTypes searchTypes, final ParsedQuery query,
+ final int limit, final int from ) {
final SearchHits searchHits = searchResponse.getHits();
final SearchHit[] hits = searchHits.getHits();
@@ -310,11 +290,10 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
candidates.add( candidateResult );
}
- final String esScrollCursor = searchResponse.getScrollId();
- final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor);
+ final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),from+limit);
// >= seems odd. However if we get an overflow, we need to account for it.
- if ( esScrollCursor != null && length >= limit ) {
+ if ( length >= limit ) {
final String cursor = candidateResults.initializeCursor();
//now set this into our map module
@@ -323,13 +302,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final int storageSeconds = ( int ) TimeUnit.MINUTES.toSeconds( minutes );
- final QueryState state = new QueryState( query.getOriginalQuery(), limit, esScrollCursor );
+ final QueryState state = new QueryState( query.getOriginalQuery(), searchEdge, searchTypes, limit, from );
final String queryStateSerialized = state.serialize();
- mapManager.putString (cursor , queryStateSerialized, storageSeconds );
+ mapManager.putString(cursor, queryStateSerialized, storageSeconds);
- logger.debug( " User cursor = {}, Cursor = {} ", cursor, esScrollCursor );
}
return candidateResults;
@@ -341,6 +319,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
*/
private static final class QueryState implements Serializable{
+
+ private static final JsonFactory SMILE_FACTORY = new JsonFactory();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
/**
* Our reserved character for constructing our storage string
*/
@@ -348,15 +331,18 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final String ql;
+ private final SearchEdge searchEdge;
+ private final SearchTypes searchTypes;
private final int limit;
+ private final int from;
- private final String esCursor;
-
- private QueryState( final String ql, final int limit, final String esCursor ) {
+ private QueryState( final String ql,final SearchEdge searchEdge, final SearchTypes searchTypes, final int limit, final int from ) {
this.ql = ql;
+ this.searchEdge = searchEdge;
+ this.searchTypes = searchTypes;
this.limit = limit;
- this.esCursor = esCursor;
+ this.from = from;
}
@@ -365,27 +351,49 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
*/
public static QueryState fromSerialized( final String input ) {
- final String[] parts = input.split( STORAGE_DELIM );
+ final String[] parts = input.split(STORAGE_DELIM);
- Preconditions.checkArgument( parts != null && parts.length == 3,
- "there must be 3 parts to the serialized query state" );
+ Preconditions.checkArgument(parts != null && parts.length == 7,
+ "there must be 5 parts to the serialized query state");
+ try {
+ String edgeName = parts[1];
+ Id edgeId = MAPPER.readValue(parts[2],SimpleId.class);
+ SearchEdge.NodeType nodeType = SearchEdge.NodeType.valueOf(parts[3]);
- return new QueryState( parts[0], Integer.parseInt( parts[1] ), parts[2] );
+ String[] searchTypes = MAPPER.readValue(parts[4],String[].class);
+ SearchTypes types = searchTypes.length == 0 ? SearchTypes.allTypes() : SearchTypes.fromTypes(searchTypes);
+ return new QueryState(parts[0], new SearchEdgeImpl(edgeId,edgeName,nodeType),types , Integer.parseInt(parts[5]), Integer.parseInt(parts[6]));
+ } catch (Exception jpe) {
+ throw new RuntimeException(jpe);
+ }
}
public String serialize() {
+ try {
- StringBuilder storageString = new StringBuilder();
+ StringBuilder storageString = new StringBuilder();
- storageString.append( ql ).append( STORAGE_DELIM );
+ storageString.append(ql).append(STORAGE_DELIM);
- storageString.append( limit ).append( STORAGE_DELIM );
+ String edge =searchEdge.getEdgeName();
+ storageString.append(edge).append(STORAGE_DELIM);
+ String nodeid = MAPPER.writeValueAsString(searchEdge.getNodeId());
+ storageString.append(nodeid).append(STORAGE_DELIM);
+ String nodeType = searchEdge.getNodeType().name();
+ storageString.append(nodeType).append(STORAGE_DELIM);
+ String types = MAPPER.writeValueAsString(searchTypes.getTypes());
+ storageString.append(types).append(STORAGE_DELIM);
- storageString.append( esCursor );
+ storageString.append(limit).append(STORAGE_DELIM);
- return storageString.toString();
+ storageString.append(from+limit);
+
+ return storageString.toString();
+ }catch (Exception jpe){
+ throw new RuntimeException(jpe);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 31c84de..33a9adf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Set;
import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.query.BoolFilterBuilder;
@@ -81,14 +82,14 @@ public class SearchRequestBuilderStrategy {
* Get the search request builder
*/
public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes searchTypes,
- final ParsedQuery query, final int limit ) {
+ final ParsedQuery query, final int limit, final int from) {
Preconditions
.checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT );
SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE )
- .setScroll( cursorTimeout + "m" );
+ esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE ).setSearchType(SearchType.QUERY_THEN_FETCH);
+
final QueryVisitor visitor = visitParsedQuery( query );
@@ -101,7 +102,7 @@ public class SearchRequestBuilderStrategy {
srb.setPostFilter( createFilterBuilder( searchEdge, visitor, searchTypes ) );
- srb = srb.setFrom( 0 ).setSize( limit );
+ srb = srb.setFrom( from ).setSize( limit );
//if we have a geo field, sort by closest to farthest by default
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 7112f9e..cd61818 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -642,8 +642,8 @@ public class EntityIndexTest extends BaseIT {
public void testCursorFormat() throws Exception {
String myType = UUID.randomUUID().toString();
- Id appId = new SimpleId( "application" );
- Id ownerId = new SimpleId( "owner" );
+ Id appId = new SimpleId(UUID.randomUUID(), "application" );
+ Id ownerId = new SimpleId( UUID.randomUUID(),"owner" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
@@ -701,31 +701,26 @@ public class EntityIndexTest extends BaseIT {
String cursor = null;
UUID lastId = null;
-
- for ( int i = 0; i < expectedPages; i++ ) {
+ final String query = "select * where mytype='"+myType+"' order by ordinal asc";
+ int i ;
+ for ( i=0; i < expectedPages; i++ ) {
//**
- final String query = "select * where mytype='"+myType+"' order by ordinal asc";
-
- final CandidateResults results =
- cursor == null ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit ) :
- entityIndex.getNextPage( cursor );
- assertTrue( results.hasCursor() );
-
- cursor = results.getCursor();
-
- assertEquals( "Should be 16 bytes as hex", 32, cursor.length() );
+ final CandidateResults results = cursor == null
+ ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, i* limit )
+ : entityIndex.getNextPage(cursor);
assertEquals(limit, results.size());
int ordinal = 0;//i == 0 ? 0 : 1;
assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
lastId = results.get(limit -1).getId().getUuid();
+ cursor = results.getCursor();
assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) );
}
//get our next page, we shouldn't get a cursor
- final CandidateResults results = entityIndex.getNextPage( cursor );
+ final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(), query, limit, i * limit);
assertEquals( 0, results.size() );
assertNull( results.getCursor() );
[08/52] [abbrv] incubator-usergrid git commit: Merge branch 'pr/180'
into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'pr/180' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8f592ddf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8f592ddf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8f592ddf
Branch: refs/heads/USERGRID-593
Commit: 8f592ddfceee61bfdf36449d421db9d856372363
Parents: 945fa08 4cb50f6
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 14:29:29 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 14:29:29 2015 -0600
----------------------------------------------------------------------
.../exceptions/DuplicateUniquePropertyExistsException.java | 2 +-
.../org/apache/usergrid/services/AbstractCollectionService.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8f592ddf/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
[27/52] [abbrv] incubator-usergrid git commit: merge
Posted by to...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a0c2651e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a0c2651e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a0c2651e
Branch: refs/heads/USERGRID-593
Commit: a0c2651e31956c7a38f39893a44a6d478d764277
Parents: f130b7c 3480a36
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 09:55:02 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 09:55:02 2015 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
stack/core/pom.xml | 8 +
.../AllApplicationsObservable.java | 35 ---
.../usergrid/corepersistence/CoreModule.java | 56 ++--
.../corepersistence/CpEntityManager.java | 273 ++++++-----------
.../corepersistence/CpEntityManagerFactory.java | 141 +++------
.../corepersistence/CpManagerCache.java | 3 +
.../corepersistence/CpRelationManager.java | 93 ++----
.../CpRelationManagerFactory.java | 46 ---
.../usergrid/corepersistence/GuiceFactory.java | 75 ++---
.../events/EntityDeletedHandler.java | 5 +-
.../events/EntityVersionCreatedHandler.java | 8 +-
.../events/EntityVersionDeletedHandler.java | 23 +-
.../index/AsyncIndexProvider.java | 122 ++++++++
.../index/AsyncReIndexService.java | 42 +++
.../index/InMemoryAsyncReIndexService.java | 89 ++++++
.../corepersistence/index/IndexEntityEvent.java | 75 +++++
.../corepersistence/index/IndexService.java | 52 ++++
.../corepersistence/index/IndexServiceImpl.java | 152 +++++++++
.../corepersistence/index/QueryFig.java | 103 +++++++
.../corepersistence/index/ReIndexAction.java | 33 ++
.../corepersistence/index/ReIndexService.java | 89 ++++++
.../index/ReIndexServiceImpl.java | 129 ++++++++
.../index/SQSAsyncReIndexService.java | 269 ++++++++++++++++
.../rx/impl/AbstractGraphVisitorImpl.java | 60 +---
.../rx/impl/AllApplicationsObservable.java | 43 +++
.../rx/impl/AllApplicationsObservableImpl.java | 5 +-
.../rx/impl/AllEntitiesInSystemImpl.java | 14 +-
.../rx/impl/AllEntityIdsObservable.java | 52 ++++
.../rx/impl/AllEntityIdsObservableImpl.java | 92 ++++++
.../rx/impl/AllNodesInGraphImpl.java | 11 +-
.../corepersistence/rx/impl/EdgeScope.java | 51 ++++
.../corepersistence/util/CpNamingUtils.java | 25 +-
.../util/SerializableMapper.java | 89 ++++++
.../usergrid/persistence/EntityManager.java | 7 -
.../persistence/EntityManagerFactory.java | 11 +-
.../persistence/GuiceAdapterBeanFactory.java | 114 +++++++
.../usergrid/persistence/PersistenceModule.java | 66 ++++
.../main/resources/usergrid-core-context.xml | 28 +-
.../corepersistence/TestIndexModule.java | 35 +++
.../corepersistence/index/PublishRxtest.java | 61 ++++
.../index/SQSAsyncIndexServiceTest.java | 167 ++++++++++
.../rx/ApplicationObservableTestIT.java | 3 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../usergrid/persistence/EntityManagerIT.java | 8 +-
.../PerformanceEntityRebuildIndexTest.java | 18 +-
.../cassandra/EntityManagerFactoryImplIT.java | 15 +-
.../resources/usergrid-custom-test.properties | 2 -
.../EntityCollectionManagerFactoryImpl.java | 15 +-
.../impl/EntityCollectionManagerImpl.java | 38 ++-
.../mvcc/stage/write/RollbackAction.java | 2 -
.../persistence/core/guice/CommonModule.java | 12 +
.../persistence/core/rx/RxSchedulerFig.java | 60 ++++
.../persistence/core/rx/RxTaskScheduler.java | 40 +++
.../core/rx/RxTaskSchedulerImpl.java | 128 ++++++++
.../core/scope/ApplicationScope.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 +++
.../persistence/core/aws/NoAWSCredsRule.java | 98 ++++++
.../graph/serialization/EdgesObservable.java | 40 ++-
.../serialization/impl/EdgesObservableImpl.java | 91 ++++--
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 6 +-
.../persistence/index/CandidateResults.java | 1 +
.../persistence/index/EntityIndexBatch.java | 3 +-
.../usergrid/persistence/index/IndexFig.java | 115 +++----
.../persistence/index/guice/IndexModule.java | 5 +-
.../persistence/index/guice/QueueProvider.java | 116 -------
.../persistence/index/impl/BufferQueue.java | 66 ----
.../index/impl/BufferQueueInMemoryImpl.java | 115 -------
.../index/impl/BufferQueueSQSImpl.java | 306 -------------------
.../impl/EsApplicationEntityIndexImpl.java | 16 +-
.../index/impl/EsEntityIndexBatchImpl.java | 16 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 52 +---
.../index/impl/EsIndexBufferConsumerImpl.java | 239 ++++++---------
.../index/impl/EsIndexBufferProducerImpl.java | 59 ----
.../index/impl/FailureMonitorImpl.java | 26 --
.../index/impl/FlushBufferQueue.java | 23 ++
.../index/impl/IndexBufferConsumer.java | 15 +-
.../index/impl/IndexBufferProducer.java | 32 --
.../persistence/index/impl/IndexIdentifier.java | 46 +++
.../index/impl/IndexIdentifierImpl.java | 118 +------
.../index/impl/IndexOperationMessage.java | 139 +++++++++
.../index/impl/IndexRefreshCommandImpl.java | 30 +-
.../persistence/index/impl/IndexingUtils.java | 6 +-
.../index/migration/LegacyIndexIdentifier.java | 4 +-
.../persistence/index/utils/StringUtils.java | 62 ----
.../persistence/index/usergrid-mappings.json | 4 +-
.../index/guice/TestIndexModule.java | 22 +-
.../index/impl/BufferQueueSQSImplTest.java | 175 -----------
.../persistence/queue/NoAWSCredsRule.java | 98 ------
.../persistence/queue/QueueManagerTest.java | 3 +-
.../usergrid/rest/management/AdminUsersIT.java | 4 +-
stack/services/pom.xml | 50 +--
.../notifications/NotifiersServiceIT.java | 19 +-
95 files changed, 3205 insertions(+), 2267 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a0c2651e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 716c7d8,e4624a9..97ac813
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@@ -27,10 -27,14 +27,11 @@@ package org.apache.usergrid.persistence
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import com.google.common.base.Optional;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import static org.apache.usergrid.persistence.core.util.StringUtils.sanitizeUUID;
++import static org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID;
/**
* Internal results class, should not be returned as results to a user.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a0c2651e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 672f090,43b55d0..89d6da7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@@ -23,13 -23,8 +23,12 @@@ package org.apache.usergrid.persistence
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
- import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
[13/52] [abbrv] incubator-usergrid git commit: check trailing page
Posted by to...@apache.org.
check trailing page
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8d965b3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8d965b3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8d965b3f
Branch: refs/heads/USERGRID-593
Commit: 8d965b3fb39a3a10c21ceb643aa4261a45449cdb
Parents: 1f83f0f
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 16:46:43 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 16:46:43 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/impl/EntityIndexTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d965b3f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 9b8a9f5..7112f9e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -720,7 +720,7 @@ public class EntityIndexTest extends BaseIT {
int ordinal = 0;//i == 0 ? 0 : 1;
assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
- lastId = results.get((i+1)*limit-1).getId().getUuid();
+ lastId = results.get(limit -1).getId().getUuid();
assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) );
}
[41/52] [abbrv] incubator-usergrid git commit: refactor cursor
Posted by to...@apache.org.
refactor cursor
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5594d745
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5594d745
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5594d745
Branch: refs/heads/USERGRID-593
Commit: 5594d745193dc6d693b8587b10b076b3d265dd79
Parents: 1674dc0
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Apr 21 15:51:01 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Apr 21 15:51:01 2015 -0600
----------------------------------------------------------------------
.../results/ElasticSearchQueryExecutor.java | 15 ++---
.../apache/usergrid/persistence/Results.java | 65 +++++++++++++-------
2 files changed, 51 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5594d745/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
index 895b457..4a1e43a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
@@ -157,12 +157,7 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
CandidateResults results = cursor.isPresent()
? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
: entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
- //set offset into query
- if(results.getOffset().isPresent()) {
- query.setOffset(results.getOffset().get());
- }else{
- query.clearOffset();
- }
+
return results;
}
@@ -187,7 +182,13 @@ public class ElasticSearchQueryExecutor implements QueryExecutor {
//signal for post processing
resultsLoader.postProcess();
- results.setCursor( query.getOffsetCursor() );
+ //set offset into query
+ if(crs.getOffset().isPresent()) {
+ query.setOffset(crs.getOffset().get());
+ }else{
+ query.clearOffset();
+ }
+ results.setCursorFromOffset( query.getOffset() );
//ugly and tight coupling, but we don't have a choice until we finish some refactoring
results.setQueryExecutor( this );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5594d745/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
index e9e7a40..e572070 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
@@ -17,19 +17,17 @@
package org.apache.usergrid.persistence;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.Base64;
import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
import javax.xml.bind.annotation.XmlRootElement;
+import com.google.common.base.*;
+import com.google.common.base.Optional;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+import org.apache.commons.codec.binary.*;
import org.apache.usergrid.corepersistence.results.QueryExecutor;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.utils.MapUtils;
@@ -46,6 +44,7 @@ import static org.apache.usergrid.utils.ConversionUtils.bytes;
@XmlRootElement
public class Results implements Iterable<Entity> {
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
Level level = Level.IDS;
@@ -173,7 +172,7 @@ public class Results implements Iterable<Entity> {
refs.add( ref( type, u ) );
}
Results r = new Results();
- r.setRefs( refs );
+ r.setRefs(refs);
return r;
}
@@ -198,14 +197,14 @@ public class Results implements Iterable<Entity> {
public static Results fromEntities( List<? extends Entity> l ) {
Results r = new Results();
- r.setEntities( l );
+ r.setEntities(l);
return r;
}
public static Results fromEntity( Entity e ) {
Results r = new Results();
- r.setEntity( e );
+ r.setEntity(e);
return r;
}
@@ -215,14 +214,14 @@ public class Results implements Iterable<Entity> {
return fromEntity( ( Entity ) ref );
}
Results r = new Results();
- r.setRef( ref );
+ r.setRef(ref);
return r;
}
public static Results fromData( Object obj ) {
Results r = new Results();
- r.setData( obj );
+ r.setData(obj);
return r;
}
@@ -231,14 +230,14 @@ public class Results implements Iterable<Entity> {
Results r = new Results();
List<AggregateCounterSet> l = new ArrayList<AggregateCounterSet>();
l.add( counters );
- r.setCounters( l );
+ r.setCounters(l);
return r;
}
public static Results fromCounters( List<AggregateCounterSet> counters ) {
Results r = new Results();
- r.setCounters( counters );
+ r.setCounters(counters);
return r;
}
@@ -246,7 +245,7 @@ public class Results implements Iterable<Entity> {
@SuppressWarnings("unchecked")
public static Results fromConnections( List<? extends ConnectionRef> connections ) {
Results r = new Results();
- r.setConnections( ( List<ConnectionRef> ) connections, true );
+ r.setConnections((List<ConnectionRef>) connections, true);
return r;
}
@@ -254,7 +253,7 @@ public class Results implements Iterable<Entity> {
@SuppressWarnings("unchecked")
public static Results fromConnections( List<? extends ConnectionRef> connections, boolean forward ) {
Results r = new Results();
- r.setConnections( ( List<ConnectionRef> ) connections, forward );
+ r.setConnections((List<ConnectionRef>) connections, forward);
return r;
}
@@ -427,7 +426,7 @@ public class Results implements Iterable<Entity> {
}
if ( ref != null ) {
refs = new ArrayList<EntityRef>();
- refs.add( ref );
+ refs.add(ref);
return refs;
}
return new ArrayList<EntityRef>();
@@ -630,7 +629,7 @@ public class Results implements Iterable<Entity> {
if ( types != null ) {
return types;
}
- getEntityRefsByType( "entity" );
+ getEntityRefsByType("entity");
if ( entitiesByType != null ) {
types = entitiesByType.keySet();
}
@@ -895,7 +894,7 @@ public class Results implements Iterable<Entity> {
public Results findForProperty( String propertyName, Object propertyValue ) {
- return findForProperty( propertyName, propertyValue, 1 );
+ return findForProperty(propertyName, propertyValue, 1);
}
@@ -916,7 +915,7 @@ public class Results implements Iterable<Entity> {
}
}
}
- return Results.fromEntities( found );
+ return Results.fromEntities(found);
}
@@ -1170,6 +1169,27 @@ public class Results implements Iterable<Entity> {
return cursor;
}
+ public Optional<Integer> getOffsetFromCursor() {
+ Optional<Integer> offset = Optional.absent();
+ if(cursor != null && cursor.length() > 0){
+ byte[] bytes = org.apache.commons.codec.binary.Base64.decodeBase64(cursor);
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
+ offset = Optional.of(number);
+ }
+ return offset;
+ }
+
+ public void setCursorFromOffset(Optional<Integer> offset) {
+
+ if(offset.isPresent()){
+ ByteBuffer buffer = INTEGER_SERIALIZER.toByteBuffer(offset.get());
+ cursor = org.apache.commons.codec.binary.Base64.encodeBase64String(buffer.array());
+ }else{
+ cursor = null;
+ }
+ }
+
public boolean hasCursor() {
return cursor != null && cursor.length() > 0;
@@ -1177,6 +1197,7 @@ public class Results implements Iterable<Entity> {
public void setCursor( String cursor ) {
+
this.cursor = cursor;
}
[16/52] [abbrv] incubator-usergrid git commit: Added equivalent
two-dot-o search/write timeouts to ES Search and Buffer Consumer.
Posted by to...@apache.org.
Added equivalent two-dot-o search/write timeouts to ES Search and Buffer Consumer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2ae6a3d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2ae6a3d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2ae6a3d1
Branch: refs/heads/USERGRID-593
Commit: 2ae6a3d1562e999af94c9f13dd5db81558d6ef0b
Parents: 078666e
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 10:06:33 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 10:06:33 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexFig.java | 15 +++++++++++++++
.../index/impl/EsApplicationEntityIndexImpl.java | 4 +++-
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index d13d055..47b25ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -71,6 +71,13 @@ public interface IndexFig extends GuicyFig {
String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ /**
+ * Timeout calls to elasticsearch.
+ * @return
+ */
+ public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+
+ public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
/**
* The client type to use. Valid values are NODE or TRANSPORT
@@ -177,5 +184,13 @@ public interface IndexFig extends GuicyFig {
@Key("elasticsearch.refresh_sleep_ms")
@Default("200")
long refreshSleep();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_QUERY_TIMEOUT )
+ long getQueryTimeout();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_WRITE_TIMEOUT )
+ long getWriteTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 0986ff6..43b55d0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -95,6 +95,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final SearchRequestBuilderStrategy searchRequest;
private FailureMonitor failureMonitor;
private final int cursorTimeout;
+ private final long queryTimeout;
@Inject
@@ -118,6 +119,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
this.searchTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.timer" );
this.cursorTimer = metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "search.cursor.timer" );
this.cursorTimeout = config.getQueryCursorTimeout();
+ this.queryTimeout = config.getWriteTimeout();
this.deleteApplicationTimer =
metricsFactory.getTimer( EsApplicationEntityIndexImpl.class, "delete.application" );
@@ -163,7 +165,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
+ searchResponse = srb.execute().actionGet(queryTimeout);
timeSearch.stop();
}
catch ( Throwable t ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ae6a3d1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 6bcc405..7cc7dcf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -209,7 +209,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
try {
- responses = bulkRequest.execute().actionGet();
+ responses = bulkRequest.execute().actionGet(indexFig.getWriteTimeout());
}
catch ( Throwable t ) {
log.error( "Unable to communicate with elasticsearch" );
[43/52] [abbrv] incubator-usergrid git commit: Updated tests and
providers for testing all async index services and index service itself
Posted by to...@apache.org.
Updated tests and providers for testing all async index services and index service itself
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f4d0d1a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f4d0d1a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f4d0d1a9
Branch: refs/heads/USERGRID-593
Commit: f4d0d1a95d8a0f5fc27eae732e61cab3d4940783
Parents: 2a58270
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Apr 22 14:23:35 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Apr 22 14:23:35 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 8 +-
.../corepersistence/CpEntityManager.java | 6 +-
.../corepersistence/CpEntityManagerFactory.java | 6 +-
.../corepersistence/CpRelationManager.java | 6 +-
.../index/AsyncIndexProvider.java | 24 +-
.../index/AsyncIndexService.java | 42 +++
.../index/AsyncReIndexService.java | 42 ---
.../index/InMemoryAsyncIndexService.java | 88 +++++
.../index/InMemoryAsyncReIndexService.java | 108 ------
.../index/IndexProcessorFig.java | 86 +++++
.../corepersistence/index/IndexService.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 9 +-
.../corepersistence/index/QueryFig.java | 103 ------
.../index/ReIndexServiceImpl.java | 12 +-
.../index/SQSAsyncIndexService.java | 325 +++++++++++++++++++
.../index/SQSAsyncReIndexService.java | 269 ---------------
.../corepersistence/util/CpNamingUtils.java | 8 +-
.../util/SerializableMapper.java | 4 +-
.../index/AsyncIndexServiceTest.java | 203 ++++++++++++
.../index/InMemoryAsycIndexServiceTest.java | 64 ++++
.../corepersistence/index/IndexServiceTest.java | 233 ++++++++++++-
.../index/SQSAsyncIndexServiceTest.java | 143 +++-----
.../impl/migration/EntityIdScope.java | 40 ++-
.../migration/data/MigrationRelationship.java | 29 +-
.../migration/data/VersionedMigrationSet.java | 31 +-
.../persistence/core/rx/RxSchedulerFig.java | 4 +-
.../core/scope/ApplicationScope.java | 4 +
.../core/scope/ApplicationScopeImpl.java | 21 +-
.../data/VersionedMigrationSetTest.java | 8 +-
.../usergrid/persistence/model/entity/Id.java | 3 +
.../persistence/model/entity/SimpleId.java | 37 ++-
.../index/impl/IndexOperationMessage.java | 8 -
32 files changed, 1259 insertions(+), 721 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 51972d8..06fe058 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -22,10 +22,10 @@ import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.corepersistence.index.AsyncIndexProvider;
-import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
-import org.apache.usergrid.corepersistence.index.QueryFig;
+import org.apache.usergrid.corepersistence.index.IndexProcessorFig;
import org.apache.usergrid.corepersistence.migration.AppInfoMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
@@ -151,9 +151,9 @@ public class CoreModule extends AbstractModule {
bind( IndexService.class ).to( IndexServiceImpl.class );
//bind the queue provider
- bind( AsyncReIndexService.class).toProvider( AsyncIndexProvider.class );
+ bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
- install( new GuicyFigModule( QueryFig.class ) );
+ install( new GuicyFigModule( IndexProcessorFig.class ) );
install( new GuicyFigModule( ApplicationIdCacheFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 72ca955..23514c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,7 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -175,7 +175,7 @@ public class CpEntityManager implements EntityManager {
private final CounterUtils counterUtils;
- private final AsyncReIndexService indexService;
+ private final AsyncIndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -215,7 +215,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncReIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 46db3f8..f08bce4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -33,7 +33,7 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
@@ -122,7 +122,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
- private final AsyncReIndexService indexService;
+ private final AsyncIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector) {
@@ -134,7 +134,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.indexService = injector.getInstance( AsyncReIndexService.class );
+ this.indexService = injector.getInstance( AsyncIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 8f125ad..ea554a3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,7 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.corepersistence.index.AsyncReIndexService;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
@@ -118,13 +118,13 @@ public class CpRelationManager implements RelationManager {
private final ApplicationScope applicationScope;
- private final AsyncReIndexService indexService;
+ private final AsyncIndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncReIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
Assert.notNull( em, "Entity manager cannot be null" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
index 18df824..0043166 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.index;
-import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
@@ -35,9 +34,9 @@ import com.google.inject.Singleton;
* A provider to allow users to configure their queue impl via properties
*/
@Singleton
-public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
+public class AsyncIndexProvider implements Provider<AsyncIndexService> {
- private final QueryFig queryFig;
+ private final IndexProcessorFig indexProcessorFig;
private final QueueManagerFactory queueManagerFactory;
private final MetricsFactory metricsFactory;
@@ -45,15 +44,15 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
private final RxTaskScheduler rxTaskScheduler;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private AsyncReIndexService asyncIndexService;
+ private AsyncIndexService asyncIndexService;
@Inject
- public AsyncIndexProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
+ public AsyncIndexProvider( final IndexProcessorFig indexProcessorFig, final QueueManagerFactory queueManagerFactory,
final MetricsFactory metricsFactory, final IndexService indexService,
final RxTaskScheduler rxTaskScheduler,
final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.queryFig = queryFig;
+ this.indexProcessorFig = indexProcessorFig;
this.queueManagerFactory = queueManagerFactory;
this.metricsFactory = metricsFactory;
this.indexService = indexService;
@@ -64,7 +63,7 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
@Override
@Singleton
- public AsyncReIndexService get() {
+ public AsyncIndexService get() {
if ( asyncIndexService == null ) {
asyncIndexService = getIndexService();
}
@@ -74,17 +73,18 @@ public class AsyncIndexProvider implements Provider<AsyncReIndexService> {
}
- private AsyncReIndexService getIndexService() {
- final String value = queryFig.getQueueImplementation();
+ private AsyncIndexService getIndexService() {
+ final String value = indexProcessorFig.getQueueImplementation();
final Implementations impl = Implementations.valueOf( value );
switch ( impl ) {
case LOCAL:
- return new InMemoryAsyncReIndexService( indexService, rxTaskScheduler,
- entityCollectionManagerFactory, metricsFactory );
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler,
+ entityCollectionManagerFactory );
case SQS:
- return new SQSAsyncReIndexService( queueManagerFactory, queryFig, metricsFactory );
+ return new SQSAsyncIndexService( queueManagerFactory, indexProcessorFig, metricsFactory, indexService,
+ entityCollectionManagerFactory, rxTaskScheduler );
default:
throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
new file mode 100644
index 0000000..dac92c8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface AsyncIndexService extends ReIndexAction {
+
+
+ /**
+ * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
+ * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
+ * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
+ * @param applicationScope
+ * @param entity The entity to index
+ */
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
deleted file mode 100644
index c6eedd7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncReIndexService.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-
-/**
- * Low level queue service for indexing entities
- */
-public interface AsyncReIndexService extends ReIndexAction {
-
-
- /**
- * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
- * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
- * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
- * @param applicationScope
- * @param entity The entity to index
- */
- void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
new file mode 100644
index 0000000..e8d178c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+
+
+@Singleton
+public class InMemoryAsyncIndexService implements AsyncIndexService {
+
+ private static final Logger log = LoggerFactory.getLogger( InMemoryAsyncIndexService.class );
+
+ private final IndexService indexService;
+ private final RxTaskScheduler rxTaskScheduler;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public InMemoryAsyncIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.indexService = indexService;
+ this.rxTaskScheduler = rxTaskScheduler;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity ) {
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+
+
+ log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
+
+ final Observable<IndexOperationMessage> edgeObservable = indexService.indexEntity( applicationScope, entity );
+
+ //start it in the background on an i/o thread
+ edgeObservable.subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
+ }
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+
+ final Id entityId = entityIdScope.getId();
+
+ //load the entity
+ entityCollectionManagerFactory.createCollectionManager( applicationScope ).load( entityId )
+ //perform indexing on the task scheduler and start it
+ .flatMap( entity -> indexService.indexEntity( applicationScope, entity ) )
+ .subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
deleted file mode 100644
index 4908945..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncReIndexService.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.codahale.metrics.Timer;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-import rx.Observable;
-
-
-@Singleton
-public class InMemoryAsyncReIndexService implements AsyncReIndexService {
-
- private static final Logger log = LoggerFactory.getLogger( InMemoryAsyncReIndexService.class );
- private final IndexService indexService;
- private final RxTaskScheduler rxTaskScheduler;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final Timer timer;
-
-
- @Inject
- public InMemoryAsyncReIndexService( final IndexService indexService, final RxTaskScheduler rxTaskScheduler,
- final EntityCollectionManagerFactory entityCollectionManagerFactory, final
- MetricsFactory metricsFactory ) {
- this.indexService = indexService;
- this.rxTaskScheduler = rxTaskScheduler;
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
-
- timer = metricsFactory.getTimer( InMemoryAsyncReIndexService.class, "IndexTimer" );
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity ) {
-
- //process the entity immediately
- //only process the same version, otherwise ignore
-
-
- log.debug( "Indexing entity {} in app scope {} ", entity, applicationScope );
-
- final Observable<IndexOperationMessage> edgeObservable = indexService.indexEntity( applicationScope, entity );
-
-
-
- edgeObservable.subscribeOn( rxTaskScheduler.getAsyncIOScheduler() ).subscribe();
-
- //now start it
-// final Timer.Context time = timer.time();
-//
-// edgeObservable.connect();
-//
-// time.stop();
-
-
- }
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
-
- final Id entityId = entityIdScope.getId();
-
- final Entity entity =
- entityCollectionManagerFactory.createCollectionManager( applicationScope ).load( entityId ).toBlocking()
- .lastOrDefault( null );
-
-
- if ( entity == null ) {
- log.warn( "Could not find entity with id {} in app scope {} ", entityId, applicationScope );
- }
-
- indexService.indexEntity( applicationScope, entity );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
new file mode 100644
index 0000000..1e8abff
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ * Application id cache fig
+ */
+public interface IndexProcessorFig extends GuicyFig {
+
+
+ /**
+ * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
+ * backpressure
+ */
+ String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
+
+ /**
+ * The number of worker threads to consume from the queue
+ */
+ String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
+
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
+
+ /**
+ * Amount of time to wait when reading from the queue
+ */
+ String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
+
+
+
+ @Default( "1000" )
+ @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
+ long getFailureRetryTime();
+
+ //give us 60 seconds to process the message
+ @Default( "60" )
+ @Key( INDEX_QUEUE_READ_TIMEOUT )
+ int getIndexQueueTimeout();
+
+ @Default( "1" )
+ @Key( ELASTICSEARCH_WORKER_COUNT )
+ int getWorkerCount();
+
+ @Default( "LOCAL" )
+ @Key( ELASTICSEARCH_QUEUE_IMPL )
+ String getQueueImplementation();
+
+
+ @Default("30000")
+ @Key("elasticsearch.reindex.sample.interval")
+ long getReIndexSampleInterval();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index 5e9392b..18ab2b7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -41,9 +41,9 @@ public interface IndexService {
* @param applicationScope The scope of the entity
* @param entity The entity
*
- * @return A ConnectableObservable with every edge in the batch to index the entity. Note that this a cold observable
- * and must be subscribed to, then "connect" in order to perform the operation. This also makes no assumptions on scheduling. It is up to the caller
- * to assign the correct scheduler to the observable
+ * @return An Observable with executed batch future as an observable. Note that this a cold observable
+ * and must be subscribed to in order to perform the index operations. This also makes no assumptions on scheduling. It is up to the caller
+ * to assign the correct scheduler to the observable based on their threading needs
*/
Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope, final Entity entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 87e22f4..81bf6cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -99,7 +99,7 @@ public class IndexServiceImpl implements IndexService {
//we might or might not need to index from target-> source
- final Observable<IndexEdge> targetSizes = getIndexEdgesToTarget( gm, entityId );
+ final Observable<IndexEdge> targetSizes = getIndexEdgesAsTarget( gm, entityId );
//merge the edges together
@@ -129,12 +129,13 @@ public class IndexServiceImpl implements IndexService {
/**
- * Get index edges to the target
+ * Get index edges to the target. Used in only certain entity types, such as roles, users, groups etc
+ * where we doubly index on both directions of the edge
*
* @param graphManager The graph manager
- * @param entityId The entitie's id
+ * @param entityId The entity's id
*/
- private Observable<IndexEdge> getIndexEdgesToTarget( final GraphManager graphManager, final Id entityId ) {
+ private Observable<IndexEdge> getIndexEdgesAsTarget( final GraphManager graphManager, final Id entityId ) {
final String collectionName = InflectionUtils.pluralize( entityId.getType() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
deleted file mode 100644
index 82ed496..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueryFig.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.safehaus.guicyfig.Default;
-import org.safehaus.guicyfig.GuicyFig;
-import org.safehaus.guicyfig.Key;
-
-
-/**
- * Application id cache fig
- */
-public interface QueryFig extends GuicyFig {
-
-
- /**
- * Amount of time in milliseconds to wait when ES rejects our request before retrying. Provides simple
- * backpressure
- */
- public static final String FAILURE_REJECTED_RETRY_WAIT_TIME = "elasticsearch.rejected_retry_wait";
-
- /**
- * The number of worker threads to consume from the queue
- */
- public static final String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
-
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
-
-
- /**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
- */
- public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
-
- /**
- * Amount of time to wait when reading from the queue
- */
- public static final String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
-
- /**
- * Amount of time to wait when reading from the queue in milliseconds
- */
- public static final String INDEX_QUEUE_TRANSACTION_TIMEOUT = "elasticsearch.queue_transaction_timeout";
-
-
- String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
-
-
- @Default( "1000" )
- @Key( FAILURE_REJECTED_RETRY_WAIT_TIME )
- long getFailureRetryTime();
-
- //give us 60 seconds to process the message
- @Default( "60" )
- @Key( INDEX_QUEUE_READ_TIMEOUT )
- int getIndexQueueTimeout();
-
- @Default( "2" )
- @Key( ELASTICSEARCH_WORKER_COUNT )
- int getWorkerCount();
-
- @Default( "LOCAL" )
- @Key( ELASTICSEARCH_QUEUE_IMPL )
- String getQueueImplementation();
-
- @Default( "1000" )
- @Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
- long getQueueOfferTimeout();
-
- /**
- * size of the buffer to build up before you send results
- */
- @Default( "1000" )
- @Key( INDEX_QUEUE_SIZE )
- int getIndexQueueSize();
-
-
- @Default("30000")
- @Key("elasticsearch.reindex.sample.interval")
- long getReIndexSampleInterval();
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 3553c87..5bf7957 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -60,20 +60,20 @@ public class ReIndexServiceImpl implements ReIndexService {
private final AllApplicationsObservable allApplicationsObservable;
private final AllEntityIdsObservable allEntityIdsObservable;
- private final QueryFig queryFig;
+ private final IndexProcessorFig indexProcessorFig;
private final RxTaskScheduler rxTaskScheduler;
private final MapManager mapManager;
- private final AsyncReIndexService indexService;
+ private final AsyncIndexService indexService;
@Inject
public ReIndexServiceImpl( final AllEntityIdsObservable allEntityIdsObservable,
final MapManagerFactory mapManagerFactory,
- final AllApplicationsObservable allApplicationsObservable, final QueryFig queryFig,
- final RxTaskScheduler rxTaskScheduler, final AsyncReIndexService indexService ) {
+ final AllApplicationsObservable allApplicationsObservable, final IndexProcessorFig indexProcessorFig,
+ final RxTaskScheduler rxTaskScheduler, final AsyncIndexService indexService ) {
this.allEntityIdsObservable = allEntityIdsObservable;
this.allApplicationsObservable = allApplicationsObservable;
- this.queryFig = queryFig;
+ this.indexProcessorFig = indexProcessorFig;
this.rxTaskScheduler = rxTaskScheduler;
this.indexService = indexService;
@@ -108,7 +108,7 @@ public class ReIndexServiceImpl implements ReIndexService {
//start our sampler and state persistence
//take a sample every sample interval to allow us to resume state with minimal loss
- runningReIndex.sample( queryFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
+ runningReIndex.sample( indexProcessorFig.getReIndexSampleInterval(), TimeUnit.MILLISECONDS,
rxTaskScheduler.getAsyncIOScheduler() )
.doOnNext( edge -> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
new file mode 100644
index 0000000..d060123
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Timer;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.Subscriber;
+import rx.Subscription;
+import rx.schedulers.Schedulers;
+
+
+@Singleton
+public class SQSAsyncIndexService implements AsyncIndexService {
+
+
+ private static final Logger log = LoggerFactory.getLogger( SQSAsyncIndexService.class );
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+
+ private static final int MAX_TAKE = 10;
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private final QueueManager queue;
+ private final IndexProcessorFig indexProcessorFig;
+ private final IndexService indexService;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final RxTaskScheduler rxTaskScheduler;
+
+ private final Timer readTimer;
+ private final Timer writeTimer;
+ private final Timer messageProcessingTimer;
+
+ private final Object mutex = new Object();
+
+
+ private final Counter indexErrorCounter;
+ private final AtomicLong counter = new AtomicLong();
+ private final AtomicLong inFlight = new AtomicLong();
+
+ //the actively running subscription
+ private List<Subscription> subscriptions = new ArrayList<>();
+
+
+ @Inject
+ public SQSAsyncIndexService( final QueueManagerFactory queueManagerFactory, final IndexProcessorFig indexProcessorFig,
+ final MetricsFactory metricsFactory, final IndexService indexService,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final RxTaskScheduler rxTaskScheduler ) {
+
+ this.indexService = indexService;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.rxTaskScheduler = rxTaskScheduler;
+
+ final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.indexProcessorFig = indexProcessorFig;
+
+ this.writeTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "write" );
+ this.readTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "read" );
+ this.messageProcessingTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "message.processing" );
+ this.indexErrorCounter = metricsFactory.getCounter( SQSAsyncIndexService.class, "error" );
+
+
+ //wire up the gauge of inflight messages
+ metricsFactory.addGauge( SQSAsyncIndexService.class, "inflight.meter", new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return inFlight.longValue();
+ }
+ } );
+
+ start();
+ }
+
+
+ /**
+ * Offer the EntityIdScope to SQS
+ */
+ private void offer( final EntityIdScope operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+
+ try {
+ //signal to SQS
+ this.queue.sendMessage( operation );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ /**
+ * Take messages from SQS
+ */
+ public List<QueueMessage> take() {
+
+ //SQS doesn't support more than 10
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+ return queue.getMessages( MAX_TAKE, indexProcessorFig.getIndexQueueTimeout(), indexProcessorFig.getIndexQueueTimeout(),
+ EntityIdScope.class );
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ /**
+ * Ack messages in SQS
+ */
+ public void ack( final List<QueueMessage> messages ) {
+
+ /**
+ * No op
+ */
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ queue.commitMessages( messages );
+ }
+
+
+ @Override
+ public void index( final EntityIdScope entityIdScope ) {
+ //queue the re-inex operation
+ offer( entityIdScope );
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity ) {
+
+ //create our scope
+ final EntityIdScope entityIdScope = new EntityIdScope( applicationScope, entity.getId() );
+
+ //send it to SQS for indexing
+ index( entityIdScope );
+ }
+
+
+ /**
+ * Index an entity and return an observable of the queue message on success
+ */
+ private Observable<IndexOperationMessage> indexEntity( final QueueMessage queueMessage ) {
+ final EntityIdScope entityIdScope = ( EntityIdScope ) queueMessage.getBody();
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+
+ //run the index operation from the entity
+ return entityCollectionManager.load( entityIdScope.getId() )
+ //invoke the indexing and take the last value
+ .flatMap( entity -> indexService.indexEntity( applicationScope, entity ).last() );
+ }
+
+
+ /**
+ * Do the indexing for a list of queue messages
+ */
+ private void doIndex( final List<QueueMessage> queueMessages ) {
+ //create parallel observables to process all 10 messages
+ final Observable<Long> observable = Observable.from( queueMessages ).flatMap( ( QueueMessage queueMessage ) -> {
+ return indexEntity( queueMessage ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
+ }, MAX_TAKE ).countLong()
+
+ //remove our in flight
+ .doOnNext( count -> inFlight.addAndGet( -1 * count ) )
+
+ //do on completed ack messages. Assumes no expections were thrown
+ .doOnCompleted( () -> ack( queueMessages ) );
+
+ //wrap with our timer and fire
+ ObservableTimer.time( observable, messageProcessingTimer ).subscribe();
+ }
+
+
+ /**
+ * Loop throught and start the workers
+ */
+ public void start() {
+ final int count = indexProcessorFig.getWorkerCount();
+
+ for ( int i = 0; i < count; i++ ) {
+ startWorker();
+ }
+ }
+
+
+ /**
+ * Stop the workers
+ */
+ public void stop() {
+ synchronized ( mutex ) {
+ //stop consuming
+
+ for ( final Subscription subscription : subscriptions ) {
+ subscription.unsubscribe();
+ }
+ }
+ }
+
+
+ private void startWorker() {
+ synchronized ( mutex ) {
+
+ Observable<List<QueueMessage>> consumer =
+ Observable.create( new Observable.OnSubscribe<List<QueueMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<QueueMessage>> subscriber ) {
+
+ //name our thread so it's easy to see
+ Thread.currentThread().setName( "QueueConsumer_" + counter.incrementAndGet() );
+
+ List<QueueMessage> drainList = null;
+
+ do {
+ Timer.Context timer = readTimer.time();
+
+ try {
+ drainList = take();
+
+ //emit our list in it's entirity to hand off to a worker pool
+ subscriber.onNext( drainList );
+
+ //take since we're in flight
+ inFlight.addAndGet( drainList.size() );
+ }
+
+ catch ( Throwable t ) {
+ final long sleepTime = indexProcessorFig.getFailureRetryTime();
+
+ log.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t );
+
+ if ( drainList != null ) {
+ inFlight.addAndGet( -1 * drainList.size() );
+ }
+
+
+ try {
+ Thread.sleep( sleepTime );
+ }
+ catch ( InterruptedException ie ) {
+ //swallow
+ }
+
+ indexErrorCounter.inc();
+ }
+
+ finally{
+ timer.stop();
+ }
+ }
+ while ( true );
+ }
+ } )
+ //this won't block our read loop, just reads and proceeds
+ .doOnNext( messages -> doIndex( messages ) ).subscribeOn( Schedulers.newThread() );
+
+ //start in the background
+
+ final Subscription subscription = consumer.subscribe();
+
+ subscriptions.add( subscription );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
deleted file mode 100644
index 60a804c..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncReIndexService.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class SQSAsyncReIndexService implements AsyncReIndexService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( SQSAsyncReIndexService.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final QueryFig queryFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public SQSAsyncReIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
- final MetricsFactory metricsFactory ) {
- final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.queryFig = queryFig;
-
- this.writeTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( SQSAsyncReIndexService.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( SQSAsyncReIndexService.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- public void offer( final IndexEntityEvent operation ) {
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
- final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if ( messages.size() == 0 ) {
- return Collections.emptyList();
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String payload = getBody( message );
-
- //now see if the key was there
-
-
- final IndexEntityEvent messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- public void ack( final List<IndexEntityEvent> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexEntityEvent ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- /** Read the object from Base64 string. */
- private IndexEntityEvent fromString( String s ) throws IOException {
- IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexEntityEvent o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
-
- private String getBody( final QueueMessage message ) {
- return message.getBody().toString();
- }
-
-
- @Override
- public void index( final EntityIdScope entityIdScope ) {
-
- }
-
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexEntityEvent {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
- super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
- this.message = message;
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-
-
- @Override
- public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Entity entity) {
- throw new UnsupportedOperationException( "Implement index rebuild" );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index c42ad10..364b071 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -99,8 +99,7 @@ public class CpNamingUtils {
* Get the index scope for the edge from the source
*/
public static IndexEdge generateScopeFromSource( final Edge edge ) {
- return new IndexEdgeImpl( edge.getSourceNode(), edge.getType(), SearchEdge.NodeType.SOURCE,
- edge.getTimestamp() );
+ return new IndexEdgeImpl( edge.getSourceNode(), edge.getType(), SearchEdge.NodeType.SOURCE, edge.getTimestamp() );
}
@@ -108,8 +107,7 @@ public class CpNamingUtils {
* Get the index scope for the edge from the source
*/
public static IndexEdge generateScopeToTarget( final Edge edge ) {
- return new IndexEdgeImpl( edge.getTargetNode(), edge.getType(), SearchEdge.NodeType.TARGET,
- edge.getTimestamp() );
+ return new IndexEdgeImpl( edge.getTargetNode(), edge.getType(), SearchEdge.NodeType.TARGET, edge.getTimestamp() );
}
@@ -123,6 +121,8 @@ public class CpNamingUtils {
/**
*
+ * TODO move sourceId to ApplicationScope
+ *
* @param sourceId
* @param collectionName
* @param entityId
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
index 1ad4115..19ecf6d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/SerializableMapper.java
@@ -27,6 +27,7 @@ import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -46,6 +47,7 @@ public class SerializableMapper {
static{
MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ SMILE_FACTORY.delegateToTextual( true );
}
/**
@@ -66,9 +68,9 @@ public class SerializableMapper {
/**
* Write the value as a string
+ * @param <T>
* @param serialized
* @param clazz
- * @param <T>
* @return
*/
public static <T extends Serializable> T fromString(final String serialized, final Class<T> clazz){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
new file mode 100644
index 0000000..faef848
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
+@NotThreadSafe
+public abstract class AsyncIndexServiceTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Inject
+ public EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+ @Inject
+ public GraphManagerFactory graphManagerFactory;
+
+
+ @Inject
+ public EntityIndexFactory entityIndexFactory;
+
+
+ private AsyncIndexService asyncIndexService;
+
+
+ /**
+ * Get the async index service
+ */
+ protected abstract AsyncIndexService getAsyncIndexService();
+
+
+ @Before
+ public void setup() {
+ asyncIndexService = getAsyncIndexService();
+ }
+
+
+ @Test( timeout = 60000 )
+ public void testMessageIndexing() throws InterruptedException {
+
+
+ ApplicationScope applicationScope =
+ new ApplicationScopeImpl( new SimpleId( UUID.randomUUID(), "application" ) );
+
+
+ final Entity testEntity = new Entity( createId( "thing" ), UUIDGenerator.newTimeUUID() );
+ testEntity.setField( new StringField( "string", "foo" ) );
+
+
+ //write the entity before indexing
+ final EntityCollectionManager collectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ collectionManager.write( testEntity ).toBlocking().last();
+
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //create our collection edge
+ final Edge collectionEdge =
+ CpNamingUtils.createCollectionEdge( applicationScope.getApplication(), "things", testEntity.getId() );
+ graphManager.writeEdge( collectionEdge ).toBlocking().last();
+
+
+ /**
+ * Write 10k edges 10 at a time in parallel
+ */
+ final Edge connectionSearch = Observable.range( 0, 10000 ).flatMap( integer -> {
+ final Id connectingId = createId( "connecting" );
+ final Edge edge = CpNamingUtils.createConnectionEdge( connectingId, "likes", testEntity.getId() );
+
+ return graphManager.writeEdge( edge ).subscribeOn( Schedulers.io() );
+ }, 10 ).toBlocking().last();
+
+
+ asyncIndexService.queueEntityIndexUpdate( applicationScope, testEntity );
+
+
+ // Thread.sleep( 1000000000000l );
+
+ final ApplicationEntityIndex applicationEntityIndex =
+ entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+ final SearchEdge collectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( collectionEdge );
+
+
+ //query until it's available
+ final CandidateResults collectionResults = getResults( applicationEntityIndex, collectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, collectionResults.size() );
+
+ assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
+
+
+ final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
+
+
+ //query until it's available
+ final CandidateResults connectionResults = getResults( applicationEntityIndex, connectionSearchEdge,
+ SearchTypes.fromTypes( testEntity.getId().getType() ), "select *", 100, 1, 100 );
+
+ assertEquals( 1, connectionResults.size() );
+
+ assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
+ }
+
+
+ private CandidateResults getResults( final ApplicationEntityIndex applicationEntityIndex,
+ final SearchEdge searchEdge, final SearchTypes searchTypes, final String ql,
+ final int count, final int expectedSize, final int attempts ) {
+
+
+ for ( int i = 0; i < attempts; i++ ) {
+ final CandidateResults candidateResults =
+ applicationEntityIndex.search( searchEdge, searchTypes, "select *", 100 );
+
+ if ( candidateResults.size() == expectedSize ) {
+ return candidateResults;
+ }
+
+ try {
+ Thread.sleep( 100 );
+ }
+ catch ( InterruptedException e ) {
+ //swallow
+ }
+ }
+
+ fail( "Could not find candidates of size " + expectedSize + "after " + attempts + " attempts" );
+
+ //we'll never reach this, required for compile
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f4d0d1a9/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/InMemoryAsycIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/InMemoryAsycIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/InMemoryAsycIndexServiceTest.java
new file mode 100644
index 0000000..43968de
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/InMemoryAsycIndexServiceTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
+@NotThreadSafe
+public class InMemoryAsycIndexServiceTest extends AsyncIndexServiceTest {
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+
+ @Inject
+ public IndexService indexService;
+
+ @Inject
+ public RxTaskScheduler rxTaskScheduler;
+
+
+ @Override
+ protected AsyncIndexService getAsyncIndexService() {
+ return new InMemoryAsyncIndexService( indexService, rxTaskScheduler, entityCollectionManagerFactory );
+ }
+
+
+
+
+
+}
[10/52] [abbrv] incubator-usergrid git commit: Updated queue impl so
that we now send an async signal to index
Posted by to...@apache.org.
Updated queue impl so that we now send an async signal to index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/745f3557
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/745f3557
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/745f3557
Branch: refs/heads/USERGRID-593
Commit: 745f3557843eff4a0006fa59913f7a4dbb592dff
Parents: 6f87fff
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 17 15:13:24 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 17 15:13:24 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 20 +-
.../corepersistence/CpEntityManager.java | 97 ++----
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../corepersistence/CpRelationManager.java | 8 +-
.../index/AsyncIndexProvider.java | 114 +++++++
.../index/AsyncIndexService.java | 47 +++
.../corepersistence/index/BufferQueue.java | 68 -----
.../index/BufferQueueInMemoryImpl.java | 116 -------
.../index/BufferQueueSQSImpl.java | 306 -------------------
.../index/InMemoryAsyncIndexService.java | 75 +++++
.../corepersistence/index/IndexEntityEvent.java | 75 +++++
.../index/IndexQueueService.java | 45 ---
.../corepersistence/index/QueueProvider.java | 112 -------
.../index/SQSAsyncIndexService.java | 263 ++++++++++++++++
.../index/SQSIndexQueueServiceImpl.java | 35 ---
.../index/BufferQueueSQSImplTest.java | 179 -----------
.../index/SQSAsyncIndexServiceTest.java | 179 +++++++++++
17 files changed, 802 insertions(+), 942 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 7e8af87..4758456 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,20 +16,17 @@
package org.apache.usergrid.corepersistence;
-import org.apache.usergrid.corepersistence.index.BufferQueue;
-import org.apache.usergrid.corepersistence.index.IndexService;
-import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
-import org.apache.usergrid.corepersistence.index.QueryFig;
-import org.apache.usergrid.corepersistence.index.QueueProvider;
-import org.apache.usergrid.corepersistence.migration.*;
-import org.apache.usergrid.persistence.PersistenceModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.safehaus.guicyfig.GuicyFigModule;
-import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.corepersistence.index.AsyncIndexProvider;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.index.IndexService;
+import org.apache.usergrid.corepersistence.index.IndexServiceImpl;
+import org.apache.usergrid.corepersistence.index.QueryFig;
+import org.apache.usergrid.corepersistence.migration.AppInfoMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
@@ -37,7 +34,6 @@ import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugi
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllNodesInGraphImpl;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -47,12 +43,12 @@ import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.graph.serialization.impl.migration.GraphNode;
import org.apache.usergrid.persistence.index.guice.IndexModule;
import com.google.inject.AbstractModule;
-import com.google.inject.Provider;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -151,7 +147,7 @@ public class CoreModule extends AbstractModule {
bind(IndexService.class).to(IndexServiceImpl.class);
//bind the queue provider
- bind( BufferQueue.class).toProvider( QueueProvider.class );
+ bind( AsyncIndexService.class).toProvider( AsyncIndexProvider.class );
install( new GuicyFigModule( QueryFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1506551..844892a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -178,7 +179,7 @@ public class CpEntityManager implements EntityManager {
private final CounterUtils counterUtils;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -218,7 +219,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final IndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
+ public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -722,7 +723,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Application app ) throws Exception {
- update(app);
+ update( app );
this.application = app;
}
@@ -903,7 +904,7 @@ public class CpEntityManager implements EntityManager {
}
- return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid()));
+ return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) );
}
@@ -970,7 +971,7 @@ public class CpEntityManager implements EntityManager {
public void setProperty(
EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
- setProperty(entityRef, propertyName, propertyValue, false);
+ setProperty( entityRef, propertyName, propertyValue, false );
}
@@ -988,7 +989,7 @@ public class CpEntityManager implements EntityManager {
entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
- entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID()));
+ entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
update( entity );
}
@@ -1083,7 +1084,7 @@ public class CpEntityManager implements EntityManager {
public void addToDictionary( EntityRef entityRef, String dictionaryName,
Object elementValue ) throws Exception {
- addToDictionary(entityRef, dictionaryName, elementValue, null);
+ addToDictionary( entityRef, dictionaryName, elementValue, null );
}
@@ -1144,7 +1145,7 @@ public class CpEntityManager implements EntityManager {
EntityRef entity = get( entityRef );
UUID timestampUuid = UUIDUtils.newTimeUUID();
- Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be);
+ Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(),
@@ -1241,7 +1242,7 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
HColumn<ByteBuffer, ByteBuffer> result =
@@ -1259,7 +1260,7 @@ public class CpEntityManager implements EntityManager {
}
}
else {
- logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null");
+ logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" );
}
return value;
@@ -1283,7 +1284,7 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
@@ -1340,7 +1341,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getDictionaries( EntityRef entity ) throws Exception {
- return getDictionaryNames(entity);
+ return getDictionaryNames( entity );
}
@@ -1386,7 +1387,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( get( entityId ))
- .getCollection(collectionName, query, resultsLevel);
+ .getCollection( collectionName, query, resultsLevel );
}
@@ -1496,7 +1497,7 @@ public class CpEntityManager implements EntityManager {
EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception {
return getRelationManager( connectingEntity )
- .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
+ .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
}
@@ -1785,14 +1786,14 @@ public class CpEntityManager implements EntityManager {
String roleTitle = batchRoleName;
String propertyName = groupId + ":" + batchRoleName;
Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
- properties.put("group", groupId);
+ properties.put( "group", groupId );
- Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties);
+ Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties );
getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) )
- .addToCollection(COLLECTION_ROLES, entity);
+ .addToCollection( COLLECTION_ROLES, entity );
- logger.info("Created role {} with id {} in group {}",
- new String[]{roleName, entity.getUuid().toString(), groupId.toString()});
+ logger.info( "Created role {} with id {} in group {}",
+ new String[] { roleName, entity.getUuid().toString(), groupId.toString() } );
return entity;
}
@@ -1883,7 +1884,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantUserPermission( UUID userId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission);
+ addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission );
}
@@ -1907,9 +1908,9 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
EntityRef userRef = userRef( userId );
EntityRef roleRef = getRoleRef( roleName );
- addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName);
- addToCollection(userRef, COLLECTION_ROLES, roleRef);
- addToCollection(roleRef, COLLECTION_USERS, userRef);
+ addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName );
+ addToCollection( userRef, COLLECTION_ROLES, roleRef );
+ addToCollection( roleRef, COLLECTION_USERS, userRef );
}
@@ -1945,7 +1946,7 @@ public class CpEntityManager implements EntityManager {
private EntityRef getRoleRef( String roleName ) throws Exception {
Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ));
+ Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.fromQL( "roleName = '" + roleName + "'" ) );
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2191,8 +2192,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getCounterNames() throws Exception {
Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
- Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS));
- names.addAll(nameSet);
+ Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) );
+ names.addAll( nameSet );
return names;
}
@@ -2264,7 +2265,7 @@ public class CpEntityManager implements EntityManager {
public Entity get( UUID uuid ) throws Exception {
MapManager mm = getMapManagerForTypes();
- String entityType = mm.getString(uuid.toString());
+ String entityType = mm.getString( uuid.toString() );
final Entity entity;
@@ -2288,7 +2289,7 @@ public class CpEntityManager implements EntityManager {
final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
- MapManager mm = managerCache.getMapManager(ms);
+ MapManager mm = managerCache.getMapManager( ms );
return mm;
}
@@ -2391,7 +2392,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
}
@@ -2406,8 +2407,8 @@ public class CpEntityManager implements EntityManager {
@Override
public void addGroupToRole( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName);
- addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName));
+ addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName );
+ addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) );
}
@@ -2428,7 +2429,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantGroupPermission( UUID groupId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission);
+ addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission );
}
@@ -2891,38 +2892,6 @@ public class CpEntityManager implements EntityManager {
} );
}
-
- void indexEntityIntoCollection( final Edge edge, final org.apache.usergrid.persistence.model.entity.Entity collectionMember) {
-
- throw new UnsupportedOperationException( "Use the new interface!" );
-//
-// final ApplicationEntityIndex aie = getManagerCache().getEntityIndex( getApplicationScope() );
-// final EntityIndexBatch batch = aie.createBatch();
-//
-// // index member into entity collection | type scope
-// final IndexEdge collectionIndexScope = generateScopeFromSource( edge );
-// batch.index( collectionIndexScope, collectionMember );
-//
-// //TODO REMOVE INDEX CODE
-// // // index member into entity | all-types scope
-// // IndexScope entityAllTypesScope = new IndexScopeImpl(
-// // collectionEntity.getId(),
-// // CpNamingUtils.ALL_TYPES, entityType );
-// //
-// // batch.index(entityAllTypesScope, memberEntity);
-// //
-// // // index member into application | all-types scope
-// // IndexScope appAllTypesScope = new IndexScopeImpl(
-// // getApplicationScope().getApplication(),
-// // CpNamingUtils.ALL_TYPES, entityType );
-// //
-// // batch.index(appAllTypesScope, memberEntity);
-//
-// //Adding graphite metrics
-// Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
-// batch.execute();
-// timeIndexEntityCollection.stop();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6ad8616..da1febc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -40,6 +40,7 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
@@ -111,7 +112,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Injector injector;
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector) {
@@ -123,7 +124,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
- this.indexService = injector.getInstance( IndexService.class );
+ this.indexService = injector.getInstance( AsyncIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index a0a44b3..db9816e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
@@ -120,13 +121,13 @@ public class CpRelationManager implements RelationManager {
private final ApplicationScope applicationScope;
- private final IndexService indexService;
+ private final AsyncIndexService indexService;
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final IndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
Assert.notNull( em, "Entity manager cannot be null" );
@@ -481,7 +482,8 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- ( ( CpEntityManager ) em ).indexEntityIntoCollection( edge, memberEntity );
+ indexService.queueEntityIndexUpdate( applicationScope, memberEntity.getId(), memberEntity.getVersion() );
+
if ( logger.isDebugEnabled() ) {
logger.debug( "Added entity {}:{} to collection {}", new Object[] {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
new file mode 100644
index 0000000..8257c94
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexProvider.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+
+
+/**
+ * A provider to allow users to configure their queue impl via properties
+ */
+@Singleton
+public class AsyncIndexProvider implements Provider<AsyncIndexService> {
+
+ private final QueryFig queryFig;
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final QueueManagerFactory queueManagerFactory;
+ private final MetricsFactory metricsFactory;
+ private final IndexService indexService;
+
+ private AsyncIndexService asyncIndexService;
+
+
+ @Inject
+ public AsyncIndexProvider( final QueryFig queryFig,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final QueueManagerFactory queueManagerFactory, final MetricsFactory metricsFactory,
+ final IndexService indexService ) {
+ this.queryFig = queryFig;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.queueManagerFactory = queueManagerFactory;
+ this.metricsFactory = metricsFactory;
+ this.indexService = indexService;
+ }
+
+
+ @Override
+ @Singleton
+ public AsyncIndexService get() {
+ if ( asyncIndexService == null ) {
+ asyncIndexService = getIndexService();
+ }
+
+
+ return asyncIndexService;
+ }
+
+
+ private AsyncIndexService getIndexService() {
+ final String value = queryFig.getQueueImplementation();
+
+ final Implementations impl = Implementations.valueOf( value );
+
+ switch ( impl ) {
+ case LOCAL:
+ return new InMemoryAsyncIndexService( indexService, entityCollectionManagerFactory );
+ case SQS:
+ return new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ default:
+ throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
+ }
+ }
+
+
+ private String getErrorValues() {
+ String values = "";
+
+ for ( final Implementations impl : Implementations.values() ) {
+ values += impl + ", ";
+ }
+
+ values = values.substring( 0, values.length() - 2 );
+
+ return values;
+ }
+
+
+ /**
+ * Different implementations
+ */
+ public static enum Implementations {
+ LOCAL,
+ SQS;
+
+
+ public String asString() {
+ return toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
new file mode 100644
index 0000000..d1f2fb6
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/AsyncIndexService.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Low level queue service for indexing entities
+ */
+public interface AsyncIndexService {
+
+
+ /**
+ * Queue an entity to be indexed. This will start processing immediately. For implementations that are realtime (akka, in memory)
+ * We will return a distributed future. For SQS impls, this will return immediately, and the result will not be available.
+ * After SQS is removed, the tests should be enhanced to ensure that we're processing our queues correctly.
+ * @param applicationScope
+ * @param entityId
+ * @param version
+ */
+ void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId, final UUID version );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
deleted file mode 100644
index fc1bdb7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueue.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-
-
-/**
- * A temporary interface of our buffer Q to decouple of producer and consumer;
- */
-public interface BufferQueue {
-
- /**
- * Offer the indexoperation message. Some queues may support not returning the future until ack or fail.
- * Other queues may return the future after ack on the offer. See the implementation documentation for details.
- * @param operation
- */
- public void offer(final IndexOperationMessage operation);
-
-
- /**
- * Perform a take, potentially blocking until up to takesize is available, or timeout has elapsed.
- * May return less than the take size, but will never return null
- *
- * @param takeSize
- * @param timeout
- * @param timeUnit
- * @return A null safe lid
- */
- public List<IndexOperationMessage> take(final int takeSize, final long timeout, final TimeUnit timeUnit );
-
-
- /**
- * Ack all messages so they do not appear again. Meant for transactional queues, and may or may not be implemented.
- * This will set the future as done in in memory operations
- *
- * @param messages
- */
- public void ack(final List<IndexOperationMessage> messages);
-
- /**
- * Mark these message as failed. Set the exception in the future on local operation
- *
- * @param messages
- */
- public void fail(final List<IndexOperationMessage> messages, final Throwable t);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
deleted file mode 100644
index 0e43da3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueInMemoryImpl.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class BufferQueueInMemoryImpl implements BufferQueue {
-
-
- private final QueryFig fig;
- private final ArrayBlockingQueue<IndexOperationMessage> messages;
-
-
- @Inject
- public BufferQueueInMemoryImpl( final QueryFig fig ) {
- this.fig = fig;
- messages = new ArrayBlockingQueue<>( fig.getIndexQueueSize() );
- }
-
-
- @Override
- public void offer( final IndexOperationMessage operation ) {
- try {
- messages.offer( operation, fig.getQueueOfferTimeout(), TimeUnit.MILLISECONDS );
- }
- catch ( InterruptedException e ) {
- throw new RuntimeException("Unable to offer message to queue", e);
- }
- }
-
-
- @Override
- public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- final List<IndexOperationMessage> response = new ArrayList<>( takeSize );
- try {
-
-
- messages.drainTo( response, takeSize );
-
- //we got something, go process it
- if ( response.size() > 0 ) {
- return response;
- }
-
-
- final IndexOperationMessage polled = messages.poll( timeout, timeUnit );
-
- if ( polled != null ) {
- response.add( polled );
-
- //try to add more
- messages.drainTo( response, takeSize - 1 );
- }
- }
- catch ( InterruptedException e ) {
- //swallow
- }
-
-
- return response;
- }
-
-
- @Override
- public void ack( final List<IndexOperationMessage> messages ) {
- //if we have a future ack it
- for ( final IndexOperationMessage op : messages ) {
- op.done();
- }
- }
-
-
- @Override
- public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
-
-
- for ( final IndexOperationMessage op : messages ) {
- final BetterFuture<IndexOperationMessage> future = op.getFuture();
-
- if ( future != null ) {
- future.setError( t );
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
deleted file mode 100644
index d955014..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImpl.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.queue.QueueManager;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.persistence.queue.QueueScope;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * This is experimental at best. Our SQS size limit is a problem. We shouldn't use this for index operation. Only for
- * performing
- */
-@Singleton
-public class BufferQueueSQSImpl implements BufferQueue {
-
- private static final Logger logger = LoggerFactory.getLogger( BufferQueueSQSImpl.class );
-
- /** Hacky, copied from CPEntityManager b/c we can't access it here */
- public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
-
-
- /**
- * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
- */
- public static final int TTL = 60 * 60 * 24 * 30;
-
- /**
- * The name to put in the map
- */
- public static final String MAP_NAME = "esqueuedata";
-
-
- private static final String QUEUE_NAME = "es_queue";
-
- private static SmileFactory SMILE_FACTORY = new SmileFactory();
-
-
- static {
- SMILE_FACTORY.delegateToTextual( true );
- }
-
-
- private final QueueManager queue;
- private final MapManager mapManager;
- private final QueryFig queryFig;
- private final ObjectMapper mapper;
- private final Meter readMeter;
- private final Timer readTimer;
- private final Meter writeMeter;
- private final Timer writeTimer;
-
-
- @Inject
- public BufferQueueSQSImpl( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- final QueueScope queueScope =
- new QueueScopeImpl( QUEUE_NAME );
-
- this.queue = queueManagerFactory.getQueueManager( queueScope );
- this.queryFig = queryFig;
-
- final MapScope scope = new MapScopeImpl( new SimpleId( MANAGEMENT_APPLICATION_ID, "application" ), MAP_NAME );
-
- this.mapManager = mapManagerFactory.createMapManager( scope );
-
-
- this.writeTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "write.timer" );
- this.writeMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "write.meter" );
-
- this.readTimer = metricsFactory.getTimer( BufferQueueSQSImpl.class, "read.timer" );
- this.readMeter = metricsFactory.getMeter( BufferQueueSQSImpl.class, "read.meter" );
-
- this.mapper = new ObjectMapper( SMILE_FACTORY );
- //pretty print, disabling for speed
- // mapper.enable(SerializationFeature.INDENT_OUTPUT);
-
- }
-
-
- @Override
- public void offer( final IndexOperationMessage operation ) {
-
- //no op
- if(operation.isEmpty()){
- operation.getFuture().done();
- return;
- }
-
- final Timer.Context timer = this.writeTimer.time();
- this.writeMeter.mark();
-
- final UUID identifier = UUIDGenerator.newTimeUUID();
-
- try {
-
- final String payLoad = toString( operation );
-
- //write to cassandra
- this.mapManager.putString( identifier.toString(), payLoad, TTL );
-
- //signal to SQS
- this.queue.sendMessage( identifier );
- operation.done();
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to queue message", e );
- }
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public List<IndexOperationMessage> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
-
- //SQS doesn't support more than 10
-
- final int actualTake = Math.min( 10, takeSize );
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
-
- List<QueueMessage> messages = queue
- .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
- String.class );
-
-
-
- final List<IndexOperationMessage> response = new ArrayList<>( messages.size() );
-
- final List<String> mapEntries = new ArrayList<>( messages.size() );
-
-
- if(messages.size() == 0){
- return response;
- }
-
- //add all our keys for a single round trip
- for ( final QueueMessage message : messages ) {
- mapEntries.add( message.getBody().toString() );
- }
-
- //look up the values
- final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
-
-
- //load them into our response
- for ( final QueueMessage message : messages ) {
-
- final String key = getMessageKey( message );
-
- //now see if the key was there
- final String payload = storedCommands.get( key );
-
- //the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
- // a DLQ
-
- if ( payload == null ) {
- continue;
- }
-
- final IndexOperationMessage messageBody;
-
- try {
- messageBody = fromString( payload );
- }
- catch ( IOException e ) {
- logger.error( "Unable to deserialize message from string. This is a bug", e );
- throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
- }
-
- SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
-
- response.add( operation );
- }
-
- readMeter.mark( response.size() );
- return response;
- }
- //stop our timer
- finally {
- timer.stop();
- }
- }
-
-
- @Override
- public void ack( final List<IndexOperationMessage> messages ) {
-
- //nothing to do
- if ( messages.size() == 0 ) {
- return;
- }
-
- List<QueueMessage> toAck = new ArrayList<>( messages.size() );
-
- for ( IndexOperationMessage ioe : messages ) {
-
-
- final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
-
- final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
-
- //remove it from the map
- mapManager.delete( key );
-
- toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
- }
-
- queue.commitMessages( toAck );
- }
-
-
- @Override
- public void fail( final List<IndexOperationMessage> messages, final Throwable t ) {
- //no op, just let it retry after the queue timeout
- }
-
-
- /** Read the object from Base64 string. */
- private IndexOperationMessage fromString( String s ) throws IOException {
- IndexOperationMessage o = mapper.readValue( s, IndexOperationMessage.class );
- return o;
- }
-
-
- /** Write the object to a Base64 string. */
- private String toString( IndexOperationMessage o ) throws IOException {
- return mapper.writeValueAsString( o );
- }
-
- private String getMessageKey(final QueueMessage message){
- return message.getBody().toString();
- }
-
- /**
- * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
- */
- public class SqsIndexOperationMessage extends IndexOperationMessage {
-
- private final QueueMessage message;
-
-
- public SqsIndexOperationMessage( final QueueMessage message, final IndexOperationMessage source ) {
- this.message = message;
- this.addAllDeIndexRequest( source.getDeIndexRequests() );
- this.addAllIndexRequest( source.getIndexRequests() );
- }
-
-
- /**
- * Get the message from our queue
- */
- public QueueMessage getMessage() {
- return message;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
new file mode 100644
index 0000000..fdc9c65
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/InMemoryAsyncIndexService.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+
+@Singleton
+public class InMemoryAsyncIndexService implements AsyncIndexService {
+
+ private final IndexService indexService;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public InMemoryAsyncIndexService( final IndexService indexService,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {this.indexService = indexService;
+
+
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
+ final UUID version ) {
+
+ final IndexEntityEvent event = new IndexEntityEvent( applicationScope, entityId, version );
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+
+ getEntity( applicationScope, entityId).filter( entity-> version.equals(entity.hasVersion() )).doOnNext( entity -> {
+ indexService.indexEntity( applicationScope, entity );
+ } ).subscribeOn( Schedulers.io() ).subscribe();
+
+
+ }
+
+ private Observable<Entity> getEntity( final ApplicationScope applicationScope, final Id entityId){
+
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+ return ecm.load( entityId );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
new file mode 100644
index 0000000..9f81f34
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexEntityEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.Serializable;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * The immutable serializable event that represents and index operations
+ */
+public class IndexEntityEvent implements Serializable {
+
+ public ApplicationScope applicationScope;
+ public Id entityId;
+ public UUID entityVersion;
+
+
+ public IndexEntityEvent( final ApplicationScope applicationScope, final Id entityId, final UUID entityVersion ) {
+ this.applicationScope = applicationScope;
+ this.entityId = entityId;
+ this.entityVersion = entityVersion;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+ public void setApplicationScope( final ApplicationScope applicationScope ) {
+ this.applicationScope = applicationScope;
+ }
+
+
+ public Id getEntityId() {
+ return entityId;
+ }
+
+
+ public void setEntityId( final Id entityId ) {
+ this.entityId = entityId;
+ }
+
+
+ public UUID getEntityVersion() {
+ return entityVersion;
+ }
+
+
+ public void setEntityVersion( final UUID entityVersion ) {
+ this.entityVersion = entityVersion;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
deleted file mode 100644
index 8d7f222..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexQueueService.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Low level queue service for indexing entities
- */
-public interface IndexQueueService {
-
-
- /**
- * Queue an entity to be index asynchronously
- * @param applicationScope
- * @param entityId
- * @param version
- */
- void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
deleted file mode 100644
index d3920db..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/QueueProvider.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
-
-
-/**
- * A provider to allow users to configure their queue impl via properties
- */
-@Singleton
-public class QueueProvider implements Provider<BufferQueue> {
-
- private final QueryFig queryFig;
-
- private final QueueManagerFactory queueManagerFactory;
- private final MapManagerFactory mapManagerFactory;
- private final MetricsFactory metricsFactory;
-
- private BufferQueue bufferQueue;
-
-
- @Inject
- public QueueProvider( final QueryFig queryFig, final QueueManagerFactory queueManagerFactory,
- final MapManagerFactory mapManagerFactory, final MetricsFactory metricsFactory ) {
- this.queryFig = queryFig;
-
-
- this.queueManagerFactory = queueManagerFactory;
- this.mapManagerFactory = mapManagerFactory;
- this.metricsFactory = metricsFactory;
- }
-
-
- @Override
- @Singleton
- public BufferQueue get() {
- if ( bufferQueue == null ) {
- bufferQueue = getQueue();
- }
-
-
- return bufferQueue;
- }
-
-
- private BufferQueue getQueue() {
- final String value = queryFig.getQueueImplementation();
-
- final Implementations impl = Implementations.valueOf( value );
-
- switch ( impl ) {
- case LOCAL:
- return new BufferQueueInMemoryImpl( queryFig );
- case SQS:
- return new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
- default:
- throw new IllegalArgumentException( "Configuration value of " + getErrorValues() + " are allowed" );
- }
- }
-
-
- private String getErrorValues() {
- String values = "";
-
- for ( final Implementations impl : Implementations.values() ) {
- values += impl + ", ";
- }
-
- values = values.substring( 0, values.length() - 2 );
-
- return values;
- }
-
-
- /**
- * Different implementations
- */
- public static enum Implementations {
- LOCAL,
- SQS;
-
-
- public String asString() {
- return toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
new file mode 100644
index 0000000..cad20bd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexService.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.queue.QueueManager;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueMessage;
+import org.apache.usergrid.persistence.queue.QueueScope;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class SQSAsyncIndexService implements AsyncIndexService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( SQSAsyncIndexService.class );
+
+ /** Hacky, copied from CPEntityManager b/c we can't access it here */
+ public static final UUID MANAGEMENT_APPLICATION_ID = UUID.fromString( "b6768a08-b5d5-11e3-a495-11ddb1de66c8" );
+
+
+ /**
+ * Set our TTL to 1 month. This is high, but in the event of a bug, we want these entries to get removed
+ */
+ public static final int TTL = 60 * 60 * 24 * 30;
+
+ /**
+ * The name to put in the map
+ */
+ public static final String MAP_NAME = "esqueuedata";
+
+
+ private static final String QUEUE_NAME = "es_queue";
+
+ private static SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ static {
+ SMILE_FACTORY.delegateToTextual( true );
+ }
+
+
+ private final QueueManager queue;
+ private final QueryFig queryFig;
+ private final ObjectMapper mapper;
+ private final Meter readMeter;
+ private final Timer readTimer;
+ private final Meter writeMeter;
+ private final Timer writeTimer;
+
+
+ @Inject
+ public SQSAsyncIndexService( final QueueManagerFactory queueManagerFactory, final QueryFig queryFig,
+ final MetricsFactory metricsFactory ) {
+ final QueueScope queueScope = new QueueScopeImpl( QUEUE_NAME );
+
+ this.queue = queueManagerFactory.getQueueManager( queueScope );
+ this.queryFig = queryFig;
+
+ this.writeTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "write.timer" );
+ this.writeMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "write.meter" );
+
+ this.readTimer = metricsFactory.getTimer( SQSAsyncIndexService.class, "read.timer" );
+ this.readMeter = metricsFactory.getMeter( SQSAsyncIndexService.class, "read.meter" );
+
+ this.mapper = new ObjectMapper( SMILE_FACTORY );
+ //pretty print, disabling for speed
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+ }
+
+
+ public void offer( final IndexEntityEvent operation ) {
+ final Timer.Context timer = this.writeTimer.time();
+ this.writeMeter.mark();
+
+ final UUID identifier = UUIDGenerator.newTimeUUID();
+
+ try {
+
+ final String payLoad = toString( operation );
+
+ //signal to SQS
+ this.queue.sendMessage( identifier );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Unable to queue message", e );
+ }
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public List<IndexEntityEvent> take( final int takeSize, final long timeout, final TimeUnit timeUnit ) {
+
+ //SQS doesn't support more than 10
+
+ final int actualTake = Math.min( 10, takeSize );
+
+ final Timer.Context timer = this.readTimer.time();
+
+ try {
+
+ List<QueueMessage> messages = queue
+ .getMessages( actualTake, queryFig.getIndexQueueTimeout(), ( int ) timeUnit.toMillis( timeout ),
+ String.class );
+
+
+ final List<IndexEntityEvent> response = new ArrayList<>( messages.size() );
+
+ final List<String> mapEntries = new ArrayList<>( messages.size() );
+
+
+ if ( messages.size() == 0 ) {
+ return Collections.emptyList();
+ }
+
+ //add all our keys for a single round trip
+ for ( final QueueMessage message : messages ) {
+ mapEntries.add( message.getBody().toString() );
+ }
+
+
+ //load them into our response
+ for ( final QueueMessage message : messages ) {
+
+ final String payload = getBody( message );
+
+ //now see if the key was there
+
+
+ final IndexEntityEvent messageBody;
+
+ try {
+ messageBody = fromString( payload );
+ }
+ catch ( IOException e ) {
+ logger.error( "Unable to deserialize message from string. This is a bug", e );
+ throw new RuntimeException( "Unable to deserialize message from string. This is a bug", e );
+ }
+
+ SqsIndexOperationMessage operation = new SqsIndexOperationMessage( message, messageBody );
+
+ response.add( operation );
+ }
+
+ readMeter.mark( response.size() );
+ return response;
+ }
+ //stop our timer
+ finally {
+ timer.stop();
+ }
+ }
+
+
+ public void ack( final List<IndexEntityEvent> messages ) {
+
+ //nothing to do
+ if ( messages.size() == 0 ) {
+ return;
+ }
+
+ List<QueueMessage> toAck = new ArrayList<>( messages.size() );
+
+ for ( IndexEntityEvent ioe : messages ) {
+
+
+ final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
+
+ toAck.add( ( ( SqsIndexOperationMessage ) ioe ).getMessage() );
+ }
+
+ queue.commitMessages( toAck );
+ }
+
+
+ /** Read the object from Base64 string. */
+ private IndexEntityEvent fromString( String s ) throws IOException {
+ IndexEntityEvent o = mapper.readValue( s, IndexEntityEvent.class );
+ return o;
+ }
+
+
+ /** Write the object to a Base64 string. */
+ private String toString( IndexEntityEvent o ) throws IOException {
+ return mapper.writeValueAsString( o );
+ }
+
+
+ private String getBody( final QueueMessage message ) {
+ return message.getBody().toString();
+ }
+
+
+ /**
+ * The message that subclasses our IndexOperationMessage. holds a pointer to the original message
+ */
+ public class SqsIndexOperationMessage extends IndexEntityEvent {
+
+ private final QueueMessage message;
+
+
+ public SqsIndexOperationMessage( final QueueMessage message, final IndexEntityEvent source ) {
+ super( source.getApplicationScope(), source.getEntityId(), source.getEntityVersion() );
+ this.message = message;
+ }
+
+
+ /**
+ * Get the message from our queue
+ */
+ public QueueMessage getMessage() {
+ return message;
+ }
+ }
+
+
+ @Override
+ public void queueEntityIndexUpdate( final ApplicationScope applicationScope, final Id entityId,
+ final UUID version ) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
deleted file mode 100644
index 42f36b1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/SQSIndexQueueServiceImpl.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class SQSIndexQueueServiceImpl implements IndexQueueService {
-
- @Override
- public void queueEntityIndex( final ApplicationScope applicationScope, final Id entityId, final UUID version ) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
deleted file mode 100644
index a76a589..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/BufferQueueSQSImplTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.index;
-
-
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.corepersistence.TestIndexModule;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
-import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
-import org.apache.usergrid.persistence.index.impl.IndexRequest;
-import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
-import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
-import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-
-import com.google.inject.Inject;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
-
-
-@RunWith(EsRunner.class)
-@UseModules({ TestIndexModule.class })
-@NotThreadSafe
-public class BufferQueueSQSImplTest {
-
-
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
-
-
- @Rule
- public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
-
- @Inject
- public QueueManagerFactory queueManagerFactory;
-
- @Inject
- public QueryFig queryFig;
-
- @Inject
- public MapManagerFactory mapManagerFactory;
-
- @Inject
- public MetricsFactory metricsFactory;
-
-
- private BufferQueueSQSImpl bufferQueueSQS;
-
- @Before
- public void setup(){
- bufferQueueSQS = new BufferQueueSQSImpl( queueManagerFactory, queryFig, mapManagerFactory, metricsFactory );
- }
-
-
-
-
- @Test
- public void testMessageIndexing(){
-
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
- final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
- assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
-
- final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
-
-
- final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
-
-
- //de-index request
- final DeIndexRequest
- deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
-
-
- SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
-
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
- SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
-
-
-
-
- IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
- indexOperationMessage.addIndexRequest( indexRequest1);
- indexOperationMessage.addIndexRequest( indexRequest2);
-
- indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
- indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
-
- bufferQueueSQS.offer( indexOperationMessage );
-
- //wait for it to send to SQS
- indexOperationMessage.getFuture().get();
-
- //now get it back
-
- final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
-
- assertTrue(ops.size() > 0);
-
- final IndexOperationMessage returnedOperation = ops.get( 0 );
-
- //get the operations out
-
- final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
-
- assertTrue(indexRequestSet.contains(indexRequest1));
- assertTrue(indexRequestSet.contains(indexRequest2));
-
-
- final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
-
- assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
- assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
-
-
-
- //now ack the message
-
- bufferQueueSQS.ack( ops );
-
- }
-
- private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
- final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
-
- List<IndexOperationMessage> ops;
-
- do{
- ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
- }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
-
- return ops;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/745f3557/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
new file mode 100644
index 0000000..0977caa
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/SQSAsyncIndexServiceTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.corepersistence.TestIndexModule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.impl.DeIndexRequest;
+import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexRequest;
+import org.apache.usergrid.persistence.index.impl.SearchEdgeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
+import org.apache.usergrid.persistence.queue.QueueManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+
+import com.google.inject.Inject;
+
+import net.jcip.annotations.NotThreadSafe;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
+
+
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+@NotThreadSafe
+public class SQSAsyncIndexServiceTest {
+
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ @Rule
+ public NoAWSCredsRule noAwsCredsRule = new NoAWSCredsRule();
+
+ @Inject
+ public QueueManagerFactory queueManagerFactory;
+
+ @Inject
+ public QueryFig queryFig;
+
+
+ @Inject
+ public MetricsFactory metricsFactory;
+
+
+ private SQSAsyncIndexService bufferQueueSQS;
+
+ @Before
+ public void setup(){
+ bufferQueueSQS = new SQSAsyncIndexService( queueManagerFactory, queryFig, metricsFactory );
+ }
+
+
+
+
+ @Test
+ public void testMessageIndexing(){
+
+ fail("fix me");
+// ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+// final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+// assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
+// assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
+//
+// final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
+// final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testDoc1",request1Data );
+//
+//
+// final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
+// final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testDoc2",request2Data );
+//
+//
+// //de-index request
+// final DeIndexRequest
+// deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId3"),"name3",
+//
+//
+// SearchEdge.NodeType.SOURCE ), new SimpleId("id3"), UUID.randomUUID() );
+//
+// final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new SearchEdgeImpl(new SimpleId("testId4"),"name4",
+// SearchEdge.NodeType.SOURCE ), new SimpleId("id4"), UUID.randomUUID() );
+//
+//
+//
+//
+// IndexOperationMessage indexOperationMessage = new IndexOperationMessage();
+// indexOperationMessage.addIndexRequest( indexRequest1);
+// indexOperationMessage.addIndexRequest( indexRequest2);
+//
+// indexOperationMessage.addDeIndexRequest( deIndexRequest1 );
+// indexOperationMessage.addDeIndexRequest( deIndexRequest2 );
+//
+// bufferQueueSQS.offer( indexOperationMessage );
+//
+// //wait for it to send to SQS
+// indexOperationMessage.getFuture().get();
+//
+// //now get it back
+//
+// final List<IndexOperationMessage> ops = getResults( 20, TimeUnit.SECONDS );
+//
+// assertTrue(ops.size() > 0);
+//
+// final IndexOperationMessage returnedOperation = ops.get( 0 );
+//
+// //get the operations out
+//
+// final Set<IndexRequest> indexRequestSet = returnedOperation.getIndexRequests();
+//
+// assertTrue(indexRequestSet.contains(indexRequest1));
+// assertTrue(indexRequestSet.contains(indexRequest2));
+//
+//
+// final Set<DeIndexRequest> deIndexRequests = returnedOperation.getDeIndexRequests();
+//
+// assertTrue( deIndexRequests.contains( deIndexRequest1 ) );
+// assertTrue( deIndexRequests.contains( deIndexRequest2 ) );
+//
+//
+//
+// //now ack the message
+//
+// bufferQueueSQS.ack( ops );
+
+ }
+
+// private List<IndexOperationMessage> getResults(final long timeout, final TimeUnit timeUnit){
+// final long endTime = System.currentTimeMillis() + timeUnit.toMillis( timeout );
+//
+// List<IndexOperationMessage> ops;
+//
+// do{
+// ops = bufferQueueSQS.take( 10, 20, TimeUnit.SECONDS );
+// }while((ops == null || ops.size() == 0 ) && System.currentTimeMillis() < endTime);
+//
+// return ops;
+// }
+
+
+
+
+}