You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/31 20:31:31 UTC
[09/10] git commit: Merge remote-tracking branch 'origin/two-dot-o'
into es-client-changes
Merge remote-tracking branch 'origin/two-dot-o' into es-client-changes
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
stack/loadtests/src/test/resources/logback-test.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5676bbf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5676bbf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5676bbf4
Branch: refs/heads/es-client-changes
Commit: 5676bbf4e4399e7a1fe3df6be1c67001a3f4e634
Parents: a2f0016 b6a8bc6
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 10:49:52 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 10:49:52 2014 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +
.../corepersistence/CpEntityManagerFactory.java | 20 ++++
.../HybridEntityManagerFactory.java | 10 ++
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 10 ++
.../collection/EntityCollectionManager.java | 10 +-
.../impl/EntityCollectionManagerImpl.java | 107 ++++++++++++-------
.../serialization/SerializationFig.java | 4 -
.../collection/EntityCollectionManagerIT.java | 12 +++
.../usergrid/persistence/index/EntityIndex.java | 14 +--
.../usergrid/persistence/index/IndexFig.java | 12 +++
.../index/impl/EsEntityIndexImpl.java | 88 +++++++++------
.../impl/EntityConnectionIndexImplTest.java | 22 +++-
.../src/test/resources/logback-test.xml | 17 +--
stack/loadtests/src/test/resources/logback.xml | 3 +-
.../apache/usergrid/rest/SwaggerServlet.java | 5 +-
.../rest/filters/HealthCheckFilter.java | 68 ++++++++++++
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 ++
18 files changed, 308 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 86e7c31,cf9207a..609a987
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -50,9 -50,9 +50,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
+ import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e10ce44,e2ab857..ebf0c21
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@@ -89,7 -87,11 +93,15 @@@ public interface IndexFig extends Guicy
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
+ public int getNumberOfShards();
+
+ @Default("1")
+ @Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
+ public int numberOfReplicas();
++
+ @Default( "20" )
+ @Key( ELASTICSEARCH_FAIL_REFRESH )
+ int getFailRefreshCount();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 739ed77,1ff747f..135822e
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -132,9 -130,9 +132,9 @@@ public class EsEntityIndexImpl implemen
createMappings();
}
- AdminClient admin = client.admin();
+ AdminClient admin = esProvider.getClient().admin();
CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
- log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
// create the document, this ensures the index is ready
@@@ -171,16 -165,18 +167,20 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- client.prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
+ esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get();
+ log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
+ VERIFY_TYPE );
+ logger.info( "Successfully created new document with docId {} in index {} and type {}",
+ tempId, indexName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+ esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER )
+ .get();
+ log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}",
+ indexName, VERIFY_TYPE );
return true;
}
@@@ -256,35 -252,25 +256,35 @@@
// to ignore any fields that are not present.
final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort =
- SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( stringSort );
- log.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
- final FieldSortBuilder numberSort =
- SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( numberSort );
- log.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort =
- SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true );
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
+ .order( order ).ignoreUnmapped( true );
srb.addSort( booleanSort );
- log.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
+ logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
}
- searchResponse = srb.execute().actionGet();
+ try {
+ searchResponse = srb.execute().actionGet();
+ }
+ catch ( Throwable t ) {
- log.error( "Unable to communicate with elasticsearch" );
++ logger.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
+ }
+
+
+ failureMonitor.success();
}
else {
String scrollId = query.getCursor();
@@@ -294,21 -280,11 +294,21 @@@
if ( scrollId.endsWith( "\"" ) ) {
scrollId = scrollId.substring( 0, scrollId.length() - 1 );
}
- log.debug( "Executing query with cursor: {} ", scrollId );
+ logger.debug( "Executing query with cursor: {} ", scrollId );
- SearchScrollRequestBuilder ssrb = client.prepareSearchScroll( scrollId )
- .setScroll( cursorTimeout + "m" );
- searchResponse = ssrb.execute().actionGet();
+ SearchScrollRequestBuilder ssrb = esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
+
+ try {
+ searchResponse = ssrb.execute().actionGet();
+ }
+ catch ( Throwable t ) {
- log.error( "Unable to communicate with elasticsearch" );
++ logger.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
+ }
+
+
+ failureMonitor.success();
}
SearchHits hits = searchResponse.getHits();
@@@ -378,13 -354,13 +378,13 @@@
* For testing only.
*/
public void deleteIndex() {
- AdminClient adminClient = client.admin();
+ AdminClient adminClient = esProvider.getClient().admin();
DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
if ( response.isAcknowledged() ) {
- log.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + indexName );
}
else {
- log.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + indexName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index fada166,28b3bfb..b1c317d
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@@ -52,9 -54,10 +53,10 @@@ import org.apache.usergrid.persistence.
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertTrue;
-@RunWith( ITRunner.class )
+@RunWith( EsRunner.class )
@UseModules( { TestIndexModule.class } )
public class EntityConnectionIndexImplTest extends BaseIT {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5676bbf4/stack/loadtests/src/test/resources/logback-test.xml
----------------------------------------------------------------------