You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/31 20:31:23 UTC
[01/10] git commit: Make number of ElasticSearch shards and replicas
configurable via usergrid props.
Repository: incubator-usergrid
Updated Branches:
refs/heads/es-client-changes 278dfca26 -> d9374bed0
Make number of ElasticSearch shards and replicas configurable via usergrid props.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cb848d81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cb848d81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cb848d81
Branch: refs/heads/es-client-changes
Commit: cb848d810d1bf095bb7108965b969c49a03cbfdd
Parents: 0bfca7e
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 14:02:58 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 14:02:58 2014 -0400
----------------------------------------------------------------------
.../src/main/resources/usergrid-default.properties | 2 ++
.../org/apache/usergrid/persistence/index/IndexFig.java | 12 ++++++++++++
.../usergrid/persistence/index/impl/EsProvider.java | 2 ++
3 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb848d81/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index fb043d4..43e3920 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -59,6 +59,8 @@ elasticsearch.cluster_name=elasticsearch
elasticsearch.index_prefix=usergrid
elasticsearch.hosts=127.0.0.1
elasticsearch.port=9300
+elasticsearch.number_shards=1
+elasticsearch.number_replicas=1
index.query.limit.default=1000
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb848d81/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 142c48a..e2ab857 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -39,6 +39,10 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
+ public static final String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
+
+ public static final String ELASTICSEARCH_NUMBER_OF_REPLICAS = "elasticsearch.number_replicas";
+
public static final String QUERY_CURSOR_TIMEOUT_MINUTES = "elasticsearch.cursor_timeout.minutes";
public static final String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
@@ -82,4 +86,12 @@ public interface IndexFig extends GuicyFig {
@Default("default")
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
+
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
+ public int getNumberOfShards();
+
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
+ public int numberOfReplicas();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb848d81/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index efc61a9..a353b43 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -161,6 +161,8 @@ public class EsProvider {
.put( "client.transport.ping_timeout", 2000 ) // milliseconds
.put( "client.transport.nodes_sampler_interval", 100 )
+ .put("index.number_of_shards", fig.getNumberOfShards() )
+ .put("index.number_of_replicas", fig.numberOfReplicas() )
.put( "network.tcp.blocking", true )
.put( "node.client", true )
.put( "node.name", nodeName )
[06/10] git commit: Add new isHealthy() methods to
EntityCollectionManager and EntityIndex,
and add new HealthCheckFilter that calls them so we can fail fast when ES or
C* not available.
Posted by to...@apache.org.
Add new isHealthy() methods to EntityCollectionManager and EntityIndex, and add new HealthCheckFilter that calls them so we can fail fast when ES or C* not available.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b6a8bc62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b6a8bc62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b6a8bc62
Branch: refs/heads/es-client-changes
Commit: b6a8bc6248dfaebec33d7842feace04a0dfaf5c6
Parents: 5570b9d
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 31 11:31:32 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 31 11:31:32 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 20 ++++
.../HybridEntityManagerFactory.java | 10 ++
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 10 ++
.../collection/EntityCollectionManager.java | 10 +-
.../impl/EntityCollectionManagerImpl.java | 107 ++++++++++++-------
.../serialization/SerializationFig.java | 4 -
.../collection/EntityCollectionManagerIT.java | 12 +++
.../usergrid/persistence/index/EntityIndex.java | 14 +--
.../index/impl/EsEntityIndexImpl.java | 76 ++++++++-----
.../impl/EntityConnectionIndexImplTest.java | 22 +++-
.../src/test/resources/logback-test.xml | 19 ++++
stack/loadtests/src/test/resources/logback.xml | 3 +-
.../apache/usergrid/rest/SwaggerServlet.java | 5 +-
.../rest/filters/HealthCheckFilter.java | 68 ++++++++++++
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 ++
16 files changed, 306 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6e6b461..cf9207a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -50,6 +50,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -719,4 +720,23 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
throw new UnsupportedOperationException( "Not supported yet." );
}
+
+ @Override
+ public boolean verifyCollectionsModuleHealthy() {
+
+ CollectionScope collScope = new CollectionScopeImpl(
+ getApplicationScope(SYSTEM_APP_ID).getApplication(),
+ getApplicationScope(SYSTEM_APP_ID).getApplication(),
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( "appinfos" ));
+
+ EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope );
+ return ecm.isHealthy();
+ }
+
+ @Override
+ public boolean verifyQueryIndexModuleHealthy() {
+
+ EntityIndex ei = managerCache.getEntityIndex( getApplicationScope( SYSTEM_APP_ID ));
+ return ei.isHealthy();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 54a5dee..79c3d7d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -163,4 +163,14 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
factory.rebuildCollectionIndex(appId, collection, po);
}
+
+ @Override
+ public boolean verifyCollectionsModuleHealthy() {
+ return factory.verifyCollectionsModuleHealthy();
+ }
+
+ @Override
+ public boolean verifyQueryIndexModuleHealthy() {
+ return factory.verifyQueryIndexModuleHealthy();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index e57aa69..e1a22c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -130,6 +130,10 @@ public interface EntityManagerFactory {
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
+ public boolean verifyCollectionsModuleHealthy();
+
+ public boolean verifyQueryIndexModuleHealthy();
+
public interface ProgressObserver {
public void onProgress( EntityRef entity);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 399bccd..2654d29 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -438,4 +438,14 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
throw new UnsupportedOperationException("Not supported.");
}
+
+ @Override
+ public boolean verifyCollectionsModuleHealthy() {
+ return true;
+ }
+
+ @Override
+ public boolean verifyQueryIndexModuleHealthy() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index f976cb5..360d8e0 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -21,18 +21,14 @@ package org.apache.usergrid.persistence.collection;
import java.util.Collection;
-import java.util.UUID;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
-
import org.apache.usergrid.persistence.model.field.Field;
import rx.Observable;
/**
- *
* The operations for performing changes on an entity
- *
*/
public interface EntityCollectionManager {
@@ -83,4 +79,10 @@ public interface EntityCollectionManager {
* @return
*/
public Observable<Entity> update ( Entity entity );
+
+ /**
+ * Return true if connection query index system (Cassandra) is healthy.
+ */
+ public boolean isHealthy();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 9f3c1ea..f24334e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,8 +19,6 @@
package org.apache.usergrid.persistence.collection.impl;
-import java.net.ConnectException;
-import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -29,7 +27,6 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +35,6 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@ -53,8 +49,6 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
-import org.apache.usergrid.persistence.collection.service.UUIDService;
-import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -63,6 +57,12 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.OperationResult;
+import com.netflix.astyanax.model.ColumnFamily;
+import com.netflix.astyanax.model.CqlResult;
+import com.netflix.astyanax.serializers.StringSerializer;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import rx.Observable;
import rx.Subscriber;
@@ -78,10 +78,9 @@ import rx.schedulers.Schedulers;
*/
public class EntityCollectionManagerImpl implements EntityCollectionManager {
- private static final Logger log = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
private final CollectionScope collectionScope;
- private final UUIDService uuidService;
//start stages
@@ -97,32 +96,32 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final MarkStart markStart;
private final MarkCommit markCommit;
- private final TaskExecutor taskExecutor;
-
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
- private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+ private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
+ private final Keyspace keyspace;
+ private SerializationFig config;
@Inject
- public EntityCollectionManagerImpl(final UUIDService uuidService, @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
+ public EntityCollectionManagerImpl(
+ @Write final WriteStart writeStart,
+ @WriteUpdate final WriteStart writeUpdate,
+ final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify,
+ final WriteCommit writeCommit, final RollbackAction rollback,
+ final MarkStart markStart, final MarkCommit markCommit,
+ final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final Keyspace keyspace,
+ final SerializationFig config,
+ @Assisted final CollectionScope collectionScope
) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
-
- Preconditions.checkNotNull(uuidService, "uuidService must be defined");
-
MvccValidationUtils.validateCollectionScope(collectionScope);
this.writeStart = writeStart;
@@ -136,9 +135,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.markStart = markStart;
this.markCommit = markCommit;
- this.uuidService = uuidService;
+ this.keyspace = keyspace;
+ this.config = config;
+
this.collectionScope = collectionScope;
- this.taskExecutor = taskExecutor;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
}
@@ -187,12 +187,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull(entityId.getType(), "Entity type is required in this stage");
return Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId)).map(markStart)
- .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
- @Override
- public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- return null;
- }
- });
+ .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
+ @Override
+ public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
+ return null;
+ }
+ });
}
@@ -229,8 +229,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public void call(final Subscriber<? super EntitySet> subscriber) {
try {
- final EntitySet results =
- entitySerializationStrategy.load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+ final EntitySet results = entitySerializationStrategy.load(
+ collectionScope, entityIds, UUIDGenerator.newTimeUUID());
subscriber.onNext(results);
subscriber.onCompleted();
@@ -253,7 +253,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Id id = value == null ? null : value.getEntityId();
return id;
} catch (ConnectionException e) {
- log.error("Failed to getIdField", e);
+ logger.error("Failed to getIdField", e);
throw new RuntimeException(e);
}
}
@@ -263,7 +263,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
public Observable<Entity> update(final Entity entity) {
- log.debug("Starting update process");
+ logger.debug("Starting update process");
//do our input validation
Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
@@ -283,7 +283,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
public void call(final Entity entity) {
- log.debug("sending entity to the queue");
+ logger.debug("sending entity to the queue");
//we an update, signal the fix
@@ -301,7 +301,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public Observable<CollectionIoEvent<MvccEntity>> stageRunner(CollectionIoEvent<Entity> writeData,
WriteStart writeState) {
- return Observable.from(writeData).map(writeState).doOnNext(new Action1<CollectionIoEvent<MvccEntity>>() {
+ return Observable.from(writeData).map(writeState).doOnNext(
+ new Action1<CollectionIoEvent<MvccEntity>>() {
@Override
public void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
@@ -325,7 +326,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
-
public Observable<VersionSet> getLatestVersion(final Collection<Id> entityIds) {
return Observable.create(new Observable.OnSubscribe<VersionSet>() {
@@ -345,4 +345,31 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
});
}
+
+
+ @Override
+ public boolean isHealthy() {
+
+ try {
+ ColumnFamily<String, String> CF_SYSTEM_LOCAL = new ColumnFamily<String, String>(
+ "system.local",
+ StringSerializer.get(),
+ StringSerializer.get(),
+ StringSerializer.get());
+
+ OperationResult<CqlResult<String, String>> result = keyspace.prepareQuery(CF_SYSTEM_LOCAL)
+ .withCql("SELECT now() FROM system.local;")
+ .execute();
+
+ if ( result.getResult().getRows().size() == 1 ) {
+ return true;
+ }
+
+ } catch ( ConnectionException ex ) {
+ logger.error("Error connecting to Cassandra", ex);
+ }
+
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 15bae5c..294c7ab 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -59,20 +59,16 @@ public interface SerializationFig extends GuicyFig {
/**
* The maximum amount of entites we can load at any one time
- * @return
*/
@Key( "collection.max.load.size" )
@Default( "1000" )
int getMaxLoadSize();
-
/**
* The maximum number of bytes a serialized entity can be. Any thing beyond this is rejected
- * @return
*/
@Key( "collection.max.entry.size" )
@Default( "512" )
int getMaxEntrySize();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 93f4a7d..763d00a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -694,4 +694,16 @@ public class EntityCollectionManagerIT {
assertEquals( MvccLogEntry.State.COMPLETE, version2Log.getState() );
assertEquals( Stage.COMMITTED, version2Log.getStage() );
}
+
+
+ @Test
+ public void healthTest() {
+
+ CollectionScope context = new CollectionScopeImpl(
+ new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ assertTrue( manager.isHealthy() );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 5f4606c..fda3a53 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -36,27 +36,27 @@ public interface EntityIndex {
public void initializeIndex();
/**
- * Create the index batch
- * @return
+ * Create the index batch.
*/
public EntityIndexBatch createBatch();
/**
* Execute query in Usergrid syntax.
*/
-
public CandidateResults search(final IndexScope indexScope, Query query );
/**
- * Get the candidate results of all versions of the entity for this id
- * @param id
- * @return
+ * Get the candidate results of all versions of the entity for this id.
*/
public CandidateResults getEntityVersions(final IndexScope indexScope, Id id);
/**
- * Refresh the index
+ * Refresh the index.
*/
public void refresh();
+ /**
+ * Return true if connection query index system (ElasticSearch) is healthy.
+ */
+ public boolean isHealthy();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fc09b5a..1ff747f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -64,12 +64,16 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+import java.util.concurrent.ExecutionException;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
/**
@@ -77,7 +81,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PR
*/
public class EsEntityIndexImpl implements EntityIndex {
- private static final Logger log = LoggerFactory.getLogger( EsEntityIndexImpl.class );
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
@@ -128,13 +132,13 @@ public class EsEntityIndexImpl implements EntityIndex {
AdminClient admin = client.admin();
CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
- log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
// create the document, this ensures the index is ready
// Immediately create a document and remove it to ensure the entire cluster is ready
- // to receive documents. Occasionally we see errors. See this post:
- // http://elasticsearch-users.115913.n3.nabble.com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+ // to receive documents. Occasionally we see errors.
+ // See this post: http://s.apache.org/index-missing-exception
testNewIndex();
}
@@ -148,16 +152,13 @@ public class EsEntityIndexImpl implements EntityIndex {
/**
- * Tests writing a document to a new index to ensure it's working correctly. Comes from email
- *
- * http://elasticsearch-users.115913.n3.nabble
- * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+ * Tests writing a document to a new index to ensure it's working correctly.
+ * See this post: http://s.apache.org/index-missing-exception
*/
-
private void testNewIndex() {
- log.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", indexName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
@@ -167,14 +168,14 @@ public class EsEntityIndexImpl implements EntityIndex {
client.prepareIndex( indexName, VERIFY_TYPE, tempId )
.setSource( DEFAULT_PAYLOAD ).get();
- log.info( "Successfully created new document with docId {} in index {} and type {}",
+ logger.info( "Successfully created new document with docId {} in index {} and type {}",
tempId, indexName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
.setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- log.info( "Successfully deleted all documents in index {} and type {}",
+ logger.info( "Successfully deleted all documents in index {} and type {}",
indexName, VERIFY_TYPE );
return true;
@@ -215,8 +216,8 @@ public class EsEntityIndexImpl implements EntityIndex {
QueryBuilder qb = query.createQueryBuilder();
- if ( log.isDebugEnabled() ) {
- log.debug( "Searching index {}\n type {}\n query {} limit {}", new Object[] {
+ if ( logger.isDebugEnabled() ) {
+ logger.debug( "Searching index {}\n type {}\n query {} limit {}", new Object[] {
this.indexName, indexType, qb.toString().replace( "\n", " " ), query.getLimit()
} );
}
@@ -229,7 +230,7 @@ public class EsEntityIndexImpl implements EntityIndex {
FilterBuilder fb = query.createFilterBuilder();
if ( fb != null ) {
- log.debug( " Filter: {} ", fb.toString() );
+ logger.debug( " Filter: {} ", fb.toString() );
srb = srb.setPostFilter( fb );
}
@@ -254,19 +255,19 @@ public class EsEntityIndexImpl implements EntityIndex {
final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
.order( order ).ignoreUnmapped( true );
srb.addSort( stringSort );
- log.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
.order( order ).ignoreUnmapped( true );
srb.addSort( numberSort );
- log.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
.order( order ).ignoreUnmapped( true );
srb.addSort( booleanSort );
- log.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
}
searchResponse = srb.execute().actionGet();
@@ -279,7 +280,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( scrollId.endsWith( "\"" ) ) {
scrollId = scrollId.substring( 0, scrollId.length() - 1 );
}
- log.debug( "Executing query with cursor: {} ", scrollId );
+ logger.debug( "Executing query with cursor: {} ", scrollId );
SearchScrollRequestBuilder ssrb = client.prepareSearchScroll( scrollId )
.setScroll( cursorTimeout + "m" );
@@ -287,7 +288,7 @@ public class EsEntityIndexImpl implements EntityIndex {
}
SearchHits hits = searchResponse.getHits();
- log.debug( " Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
+ logger.debug( " Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
List<CandidateResult> candidates = new ArrayList<CandidateResult>();
@@ -307,7 +308,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( candidates.size() >= query.getLimit() ) {
candidateResults.setCursor( searchResponse.getScrollId() );
- log.debug( " Cursor = " + searchResponse.getScrollId() );
+ logger.debug( " Cursor = " + searchResponse.getScrollId() );
}
return candidateResults;
@@ -317,18 +318,18 @@ public class EsEntityIndexImpl implements EntityIndex {
public void refresh() {
- log.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", indexName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
try {
client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + indexName );
return true;
}
catch ( IndexMissingException e ) {
- log.error( "Unable to refresh index after create. Waiting before sleeping.", e);
+ logger.error( "Unable to refresh index after create. Waiting before sleeping.", e);
throw e;
}
}
@@ -336,7 +337,7 @@ public class EsEntityIndexImpl implements EntityIndex {
doInRetry( retryOperation );
- log.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + indexName );
}
@@ -356,10 +357,10 @@ public class EsEntityIndexImpl implements EntityIndex {
AdminClient adminClient = client.admin();
DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
if ( response.isAcknowledged() ) {
- log.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + indexName );
}
else {
- log.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + indexName );
}
}
@@ -377,7 +378,7 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
catch ( Exception e ) {
- log.error( "Unable to execute operation, retrying", e );
+ logger.error( "Unable to execute operation, retrying", e );
}
@@ -391,6 +392,25 @@ public class EsEntityIndexImpl implements EntityIndex {
}
+ @Override
+ public boolean isHealthy() {
+
+ try {
+ ClusterHealthResponse health =
+ client.admin().cluster().health( new ClusterHealthRequest() ).get();
+
+ if ( health.getStatus().equals( ClusterHealthStatus.GREEN ) ) {
+ return true;
+ }
+ }
+ catch (Exception ex) {
+ logger.error("Error connecting to ElasticSearch", ex);
+ }
+
+ return false ;
+ }
+
+
/**
* Interface for operations
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 62910bc..28b3bfb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -50,9 +50,11 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@@ -82,7 +84,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
// create a muffin
CollectionScope muffinScope = new CollectionScopeImpl( appId, appId, "muffins" );
- Entity muffin = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), muffinScope.getName() ) );
+ Entity muffin = new Entity(
+ new SimpleId( UUIDGenerator.newTimeUUID(), muffinScope.getName() ) );
muffin = EntityIndexMapUtils.fromMap( muffin, new HashMap<String, Object>() {{
put( "size", "Large" );
@@ -93,7 +96,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
// create a person who likes muffins
CollectionScope peopleScope = new CollectionScopeImpl( appId, appId, "people" );
- Entity person = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), peopleScope.getName() ) );
+ Entity person = new Entity( new SimpleId(
+ UUIDGenerator.newTimeUUID(), peopleScope.getName() ) );
person = EntityIndexMapUtils.fromMap( person, new HashMap<String, Object>() {{
put( "name", "Dave" );
put( "hometown", "Chapel Hill" );
@@ -121,4 +125,18 @@ public class EntityConnectionIndexImplTest extends BaseIT {
assertEquals(muffin.getId(), likes.get(0).getId());
}
+
+
+
+ @Test
+ public void healthTest() {
+
+ Id appId = new SimpleId( "application" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ EntityIndex ei = ecif.createEntityIndex( applicationScope );
+
+ assertTrue( ei.isHealthy() );
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/test/resources/logback-test.xml b/stack/loadtests/src/test/resources/logback-test.xml
index d15f9f4..f3f0607 100644
--- a/stack/loadtests/src/test/resources/logback-test.xml
+++ b/stack/loadtests/src/test/resources/logback-test.xml
@@ -1,3 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/loadtests/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/test/resources/logback.xml b/stack/loadtests/src/test/resources/logback.xml
index 1d512d6..6ba218c 100644
--- a/stack/loadtests/src/test/resources/logback.xml
+++ b/stack/loadtests/src/test/resources/logback.xml
@@ -1,5 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
<!--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,6 +17,8 @@
* limitations under the License.
*/
-->
+<configuration>
+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/rest/src/main/java/org/apache/usergrid/rest/SwaggerServlet.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SwaggerServlet.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SwaggerServlet.java
index 576063e..9db6ff4 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SwaggerServlet.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SwaggerServlet.java
@@ -41,9 +41,9 @@ import org.springframework.context.ApplicationContext;
import org.apache.commons.lang.text.StrSubstitutor;
import static org.apache.commons.lang.StringUtils.isEmpty;
-import static org.springframework.web.context.support.WebApplicationContextUtils.getRequiredWebApplicationContext;
import static org.apache.usergrid.rest.utils.CORSUtils.allowAllOrigins;
import static org.apache.usergrid.utils.StringUtils.readClasspathFileAsString;
+import org.springframework.web.context.support.WebApplicationContextUtils;
public class SwaggerServlet extends HttpServlet implements Filter {
@@ -85,7 +85,8 @@ public class SwaggerServlet extends HttpServlet implements Filter {
if ( sc == null ) {
return null;
}
- ApplicationContext appContext = getRequiredWebApplicationContext( sc );
+ ApplicationContext appContext =
+ WebApplicationContextUtils.getRequiredWebApplicationContext( sc );
return appContext.getBean( beanName );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
new file mode 100644
index 0000000..4ba9b09
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.filters;
+
+import java.io.IOException;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.springframework.web.context.WebApplicationContext;
+import org.springframework.web.context.support.WebApplicationContextUtils;
+
+
+/**
+ * Fail fast if connection to database or query index not healthy.
+ */
+public class HealthCheckFilter implements Filter {
+
+ ServletContext sc;
+
+ @Override
+ public void init(FilterConfig fc) throws ServletException {
+ if ( sc == null ) {
+ sc = fc.getServletContext();
+ }
+ }
+
+ @Override
+ public void doFilter(ServletRequest sr, ServletResponse sr1, FilterChain fc)
+ throws IOException, ServletException {
+
+
+ WebApplicationContext ctx = WebApplicationContextUtils.getRequiredWebApplicationContext(sc);
+ EntityManagerFactory emf = (EntityManagerFactory)ctx.getBean("entityManagerFactory");
+
+ if ( !emf.verifyCollectionsModuleHealthy() ) {
+ throw new RuntimeException("Error connecting to datastore");
+ }
+ if ( !emf.verifyQueryIndexModuleHealthy() ) {
+ throw new RuntimeException("Error connecting to query index");
+ }
+
+ throw new RuntimeException("Foo!");
+ }
+
+ @Override
+ public void destroy() {
+ // no op
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a8bc62/stack/rest/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/web.xml b/stack/rest/src/main/webapp/WEB-INF/web.xml
index 24a82ca..c50251b 100644
--- a/stack/rest/src/main/webapp/WEB-INF/web.xml
+++ b/stack/rest/src/main/webapp/WEB-INF/web.xml
@@ -33,6 +33,16 @@
</listener>
<filter>
+ <filter-name>healthCheckFilter</filter-name>
+ <filter-class>org.apache.usergrid.rest.filters.HealthCheckFilter</filter-class>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>healthCheckFilter</filter-name>
+ <url-pattern>/*</url-pattern>
+ </filter-mapping>
+
+ <filter>
<filter-name>swaggerFilter</filter-name>
<filter-class>org.apache.usergrid.rest.SwaggerServlet</filter-class>
</filter>
[02/10] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5570b9d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5570b9d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5570b9d3
Branch: refs/heads/es-client-changes
Commit: 5570b9d37298942515c88b69a7ba124c6b1373d7
Parents: cb848d8 aba5121
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 15:29:25 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 15:29:25 2014 -0400
----------------------------------------------------------------------
stack/loadtests/src/main/scripts/gatling-mvn.sh | 3 +++
.../src/test/resources/logback-test.xml | 22 ++++++++++++++++++++
2 files changed, 25 insertions(+)
----------------------------------------------------------------------
[07/10] git commit: Merge branch 'key-row-sharding' into
es-client-changes
Posted by to...@apache.org.
Merge branch 'key-row-sharding' into es-client-changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/37d85984
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/37d85984
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/37d85984
Branch: refs/heads/es-client-changes
Commit: 37d8598476811d8211682513cf85abd6f1cca38a
Parents: 278dfca c02e77e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 10:30:46 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 10:30:46 2014 -0600
----------------------------------------------------------------------
.../src/test/resources/logback-test.xml | 35 +++++++++++++++++++-
1 file changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[05/10] git commit: Merge branch 'key-row-sharding' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
key-row-sharding
Posted by to...@apache.org.
Merge branch 'key-row-sharding' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into key-row-sharding
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c02e77ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c02e77ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c02e77ee
Branch: refs/heads/es-client-changes
Commit: c02e77eef5316d0b220c51efacf99add9bda6fef
Parents: a5137ca ef4ede2
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 30 20:39:14 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 30 20:39:14 2014 -0600
----------------------------------------------------------------------
.../loadtests/src/test/resources/logback-test.xml | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c02e77ee/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --cc stack/loadtests/src/test/resources/logback-test.xml
index 6a188ab,d98b728..e1528ca
--- a/stack/loadtests/src/test/resources/logback-test.xml
+++ b/stack/loadtests/src/test/resources/logback-test.xml
@@@ -1,23 -1,20 +1,38 @@@
+<!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
<configuration>
-
+ <!--/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ -->
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern>
[10/10] git commit: Fixes shard allocation configuration in rest nodes
Posted by to...@apache.org.
Fixes shard allocation configuration in rest nodes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d9374bed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d9374bed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d9374bed
Branch: refs/heads/es-client-changes
Commit: d9374bed0175f05e4971880553832439d67a5403
Parents: 5676bbf
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 13:31:14 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 13:31:14 2014 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9374bed/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 7508f2e..65b82d0 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -40,6 +40,10 @@ def clusterName = System.getenv().get("CASSANDRA_CLUSTER_NAME")
def superUserEmail = System.getenv().get("SUPER_USER_EMAIL")
def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
+def numEsNodes = System.getenv().get("ES_NUM_SERVERS")
+def esShards = numEsNodes/2;
+def esReplicas = 2;
+
def cassThreads = System.getenv().get("TOMCAT_THREADS")
def hystrixThreads = Integer.parseInt(cassThreads) / 100
@@ -95,6 +99,8 @@ elasticsearch.cluster_name=${clusterName}
elasticsearch.index_prefix=usergrid
elasticsearch.hosts=${esnodes}
elasticsearch.port=9300
+elasticsearch.number_shards=${esShards}
+elasticsearch.number_replicas=${esReplicas}
######################################################
# Custom mail transport
@@ -172,6 +178,7 @@ usergrid.metrics.graphite.host=${graphite}
usergrid.queue.prefix=${clusterName}
usergrid.queue.region=${ec2Region}
+
"""
println usergridConfig
[04/10] git commit: Fixes logging in test framework
Posted by to...@apache.org.
Fixes logging in test framework
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a5137cad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a5137cad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a5137cad
Branch: refs/heads/es-client-changes
Commit: a5137cad58c4b997a4d0f7132fa1db1629f7e3a5
Parents: 0bc8b5d
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 30 20:38:12 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 30 20:38:12 2014 -0600
----------------------------------------------------------------------
.../loadtests/src/test/resources/logback-test.xml | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a5137cad/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/test/resources/logback-test.xml b/stack/loadtests/src/test/resources/logback-test.xml
index d15f9f4..6a188ab 100644
--- a/stack/loadtests/src/test/resources/logback-test.xml
+++ b/stack/loadtests/src/test/resources/logback-test.xml
@@ -1,3 +1,21 @@
+<!--
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
[03/10] git commit: fix rat warning
Posted by to...@apache.org.
fix rat warning
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef4ede29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef4ede29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef4ede29
Branch: refs/heads/es-client-changes
Commit: ef4ede29b8718a12bc908c7d2beefafcfcd01c87
Parents: 4aab84f
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 30 15:44:47 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 30 15:44:47 2014 -0600
----------------------------------------------------------------------
.../loadtests/src/test/resources/logback-test.xml | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef4ede29/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/test/resources/logback-test.xml b/stack/loadtests/src/test/resources/logback-test.xml
index d15f9f4..d98b728 100644
--- a/stack/loadtests/src/test/resources/logback-test.xml
+++ b/stack/loadtests/src/test/resources/logback-test.xml
@@ -1,5 +1,20 @@
<configuration>
-
+ <!--/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ -->
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern>
[09/10] git commit: Merge remote-tracking branch 'origin/two-dot-o'
into es-client-changes
Posted by to...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into es-client-changes
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
stack/loadtests/src/test/resources/logback-test.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5676bbf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5676bbf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5676bbf4
Branch: refs/heads/es-client-changes
Commit: 5676bbf4e4399e7a1fe3df6be1c67001a3f4e634
Parents: a2f0016 b6a8bc6
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 10:49:52 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 10:49:52 2014 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +
.../corepersistence/CpEntityManagerFactory.java | 20 ++++
.../HybridEntityManagerFactory.java | 10 ++
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 10 ++
.../collection/EntityCollectionManager.java | 10 +-
.../impl/EntityCollectionManagerImpl.java | 107 ++++++++++++-------
.../serialization/SerializationFig.java | 4 -
.../collection/EntityCollectionManagerIT.java | 12 +++
.../usergrid/persistence/index/EntityIndex.java | 14 +--
.../usergrid/persistence/index/IndexFig.java | 12 +++
.../index/impl/EsEntityIndexImpl.java | 88 +++++++++------
.../impl/EntityConnectionIndexImplTest.java | 22 +++-
.../src/test/resources/logback-test.xml | 17 +--
stack/loadtests/src/test/resources/logback.xml | 3 +-
.../apache/usergrid/rest/SwaggerServlet.java | 5 +-
.../rest/filters/HealthCheckFilter.java | 68 ++++++++++++
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 ++
18 files changed, 308 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 86e7c31,cf9207a..609a987
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -50,9 -50,9 +50,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
+ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e10ce44,e2ab857..ebf0c21
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@@ -89,7 -87,11 +93,15 @@@ public interface IndexFig extends Guicy
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
+ public int getNumberOfShards();
+
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
+ public int numberOfReplicas();
++
+ @Default( "20" )
+ @Key( ELASTICSEARCH_FAIL_REFRESH )
+ int getFailRefreshCount();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 739ed77,1ff747f..135822e
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -132,9 -130,9 +132,9 @@@ public class EsEntityIndexImpl implemen
createMappings();
}
- AdminClient admin = client.admin();
+ AdminClient admin = esProvider.getClient().admin();
CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
- log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
// create the document, this ensures the index is ready
@@@ -171,16 -165,18 +167,20 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- client.prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
+ esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get();
+ log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
+ VERIFY_TYPE );
+ logger.info( "Successfully created new document with docId {} in index {} and type {}",
+ tempId, indexName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+ esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER )
+ .get();
+ log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}",
+ indexName, VERIFY_TYPE );
return true;
}
@@@ -256,35 -252,25 +256,35 @@@
// to ignore any fields that are not present.
final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort =
- SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( stringSort );
- log.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
- final FieldSortBuilder numberSort =
- SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( numberSort );
- log.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort =
- SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( booleanSort );
- log.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
}
- searchResponse = srb.execute().actionGet();
+ try {
+ searchResponse = srb.execute().actionGet();
+ }
+ catch ( Throwable t ) {
- log.error( "Unable to communicate with elasticsearch" );
++ logger.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
+ }
+
+
+ failureMonitor.success();
}
else {
String scrollId = query.getCursor();
@@@ -294,21 -280,11 +294,21 @@@
if ( scrollId.endsWith( "\"" ) ) {
scrollId = scrollId.substring( 0, scrollId.length() - 1 );
}
- log.debug( "Executing query with cursor: {} ", scrollId );
+ logger.debug( "Executing query with cursor: {} ", scrollId );
- SearchScrollRequestBuilder ssrb = client.prepareSearchScroll( scrollId )
- .setScroll( cursorTimeout + "m" );
- searchResponse = ssrb.execute().actionGet();
+ SearchScrollRequestBuilder ssrb = esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
+
+ try {
+ searchResponse = ssrb.execute().actionGet();
+ }
+ catch ( Throwable t ) {
- log.error( "Unable to communicate with elasticsearch" );
++ logger.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
+ }
+
+
+ failureMonitor.success();
}
SearchHits hits = searchResponse.getHits();
@@@ -378,13 -354,13 +378,13 @@@
* For testing only.
*/
public void deleteIndex() {
- AdminClient adminClient = client.admin();
+ AdminClient adminClient = esProvider.getClient().admin();
DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
if ( response.isAcknowledged() ) {
- log.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + indexName );
}
else {
- log.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + indexName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index fada166,28b3bfb..b1c317d
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@@ -52,9 -54,10 +53,10 @@@ import org.apache.usergrid.persistence.
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertTrue;
-@RunWith( ITRunner.class )
+@RunWith( EsRunner.class )
@UseModules( { TestIndexModule.class } )
public class EntityConnectionIndexImplTest extends BaseIT {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------
[08/10] git commit: Changed to use transport client
Posted by to...@apache.org.
Changed to use transport client
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a2f00166
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a2f00166
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a2f00166
Branch: refs/heads/es-client-changes
Commit: a2f001663d1310117f2dcf14526283837e8af2a9
Parents: 37d8598
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 10:43:20 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 10:43:20 2014 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsProvider.java | 90 +++++++++++++-------
1 file changed, 59 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a2f00166/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index 0a06e8f..ea63ffc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -23,8 +23,10 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.slf4j.Logger;
@@ -48,7 +50,7 @@ public class EsProvider {
private final IndexFig indexFig;
private static Client client;
- private static Node node;
+// private static Node node;
public static String LOCAL_ES_PORT_PROPNAME = "EMBEDDED_ES_PORT";
@@ -76,9 +78,7 @@ public class EsProvider {
*/
public void releaseClient() {
//reset our static variables
- if ( client != null && node != null ) {
- node.stop();
- node = null;
+ if ( client != null ) {
client = null;
}
}
@@ -87,22 +87,29 @@ public class EsProvider {
private synchronized void createClient( IndexFig fig ) {
- if ( client != null && node != null) {
+ if ( client != null) {
return;
}
- String allHosts = "";
+
+ final String clusterName = fig.getClusterName();
+ final int port = fig.getPort();
+
+
+ ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
+ .put( "cluster.name", clusterName )
+ .put( "client.transport.sniff", true );
+
+
+
+// String allHosts = "";
+
- // we will connect to ES on all configured hosts
- String SEP = "";
- for ( String host : fig.getHosts().split( "," ) ) {
- allHosts = allHosts + SEP + host + ":" + fig.getPort();
- SEP = ",";
- }
String nodeName = fig.getNodeName();
+
if ( "default".equals( nodeName ) ) {
// no nodeName was specified, use hostname
try {
@@ -114,31 +121,52 @@ public class EsProvider {
}
}
- Settings settings = ImmutableSettings.settingsBuilder()
-
- .put( "cluster.name", fig.getClusterName() )
+ settings.put( "node.name", nodeName);
- // this assumes that we're using zen for host discovery. Putting an
- // explicit set of bootstrap hosts ensures we connect to a valid cluster.
- .put( "discovery.zen.ping.unicast.hosts", allHosts )
- .put( "discovery.zen.ping.multicast.enabled", "false" ).put( "http.enabled", false )
- .put( "client.transport.ping_timeout", 2000 ) // milliseconds
- .put( "client.transport.nodes_sampler_interval", 100 ).put( "network.tcp.blocking", true )
- .put( "node.client", true ).put( "node.name", nodeName )
- .build();
+ TransportClient transportClient = new TransportClient( settings.build() );
+ // we will connect to ES on all configured hosts
+// String SEP = "";
+ for ( String host : fig.getHosts().split( "," ) ) {
- log.debug( "Creating ElasticSearch client with settings: " + settings.getAsMap() );
+ transportClient.addTransportAddress( new InetSocketTransportAddress(host, port));
- // use this client when connecting via socket only,
- // such as ssh tunnel or other firewall issues
- // newClient = new TransportClient(settings).addTransportAddress(
- // new InetSocketTransportAddress("localhost", 9300) );
+// allHosts = allHosts + SEP + host + ":" + fig.getPort();
+// SEP = ",";
+ }
- //use this client for quick connectivity
- node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).node();
- client = node.client();
+// Settings settings = ImmutableSettings.settingsBuilder()
+//
+// .put( "cluster.name", fig.getClusterName() )
+//
+// // this assumes that we're using zen for host discovery. Putting an
+// // explicit set of bootstrap hosts ensures we connect to a valid cluster.
+// .put( "discovery.zen.ping.unicast.hosts", allHosts )
+// .put( "discovery.zen.ping.multicast.enabled", "false" ).put( "http.enabled", false )
+// .put("transport.tcp.port", "9300-9400")
+// .put( "client.transport.ping_timeout", 2000 ) // milliseconds
+// .put( "client.transport.nodes_sampler_interval", 100 )
+//// .put( "network.tcp.blocking", true )
+// .put( "node.client", true ).put( "node.name", nodeName+"Random test" )
+//
+// .build();
+
+// log.debug( "Creating ElasticSearch client with settings: " + settings.getAsMap() );
+//
+// // use this client when connecting via socket only,
+// // such as ssh tunnel or other firewall issues
+// // newClient = new TransportClient(settings).addTransportAddress(
+// // new InetSocketTransportAddress("localhost", 9300) );
+//
+// //use this client for quick connectivity
+// node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).node();
+// client = node.client();
+
+
+
+// .build();
+ client = transportClient;
}