You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/11/21 22:44:37 UTC
[28/28] incubator-usergrid git commit: merge from two-o
merge from two-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3ff891e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3ff891e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3ff891e1
Branch: refs/heads/index-alias
Commit: 3ff891e159f2caf61a356c1f0fced387ac17595a
Parents: 55110a5 0a92bee
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 14:43:38 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 14:43:38 2014 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 32 +-
.../corepersistence/CpEntityDeleteListener.java | 9 +-
.../corepersistence/CpEntityManager.java | 44 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpRelationManager.java | 35 +-
.../usergrid/corepersistence/GuiceModule.java | 2 +
.../migration/EntityDataMigration.java | 147 ++
.../migration/EntityTypeMappingMigration.java | 28 +-
.../migration/GraphShardVersionMigration.java | 104 +-
.../corepersistence/migration/Versions.java | 3 +
.../rx/AllEntitiesInSystemObservable.java | 27 +-
.../rx/ApplicationObservable.java | 65 +-
.../corepersistence/util/CpNamingUtils.java | 17 +
.../org/apache/usergrid/CoreITSetupImpl.java | 13 +
.../CpEntityDeleteListenerTest.java | 4 +-
.../corepersistence/StaleIndexCleanupTest.java | 5 +-
.../migration/EntityDataMigrationIT.java | 251 ++++
.../migration/EntityTypeMappingMigrationIT.java | 74 +-
.../migration/GraphShardVersionMigrationIT.java | 153 +-
.../rx/AllEntitiesInSystemObservableIT.java | 23 +-
.../org/apache/usergrid/persistence/GeoIT.java | 404 ++++--
.../usergrid/persistence/LargeEntityIT.java | 112 ++
stack/core/src/test/resources/largeentity.json | 1329 ++++++++++++++++++
.../exception/EntityTooLargeException.java | 67 +
.../impl/EntityCollectionManagerImpl.java | 251 ++--
.../mvcc/MvccEntitySerializationStrategy.java | 23 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 3 +-
.../mvcc/stage/delete/MarkCommit.java | 5 +-
.../mvcc/stage/write/WriteCommit.java | 3 +-
.../serialization/SerializationFig.java | 15 +-
.../UniqueValueSerializationStrategy.java | 3 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 343 ++---
...vccEntitySerializationStrategyProxyImpl.java | 162 +++
.../MvccEntitySerializationStrategyV1Impl.java | 219 +++
.../MvccEntitySerializationStrategyV2Impl.java | 238 ++++
.../MvccLogEntrySerializationStrategyImpl.java | 2 +-
.../serialization/impl/SerializationModule.java | 25 +-
.../serialization/impl/SettingsValidation.java | 58 +
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../EntityCollectionManagerFactoryTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 64 +-
.../EntityCollectionManagerStressTest.java | 2 +-
.../EntityCollectionManagerSyncIT.java | 2 +-
.../collection/guice/MigrationManagerRule.java | 38 -
.../collection/guice/TestCollectionModule.java | 14 +
...niqueValueSerializationStrategyImplTest.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../serialization/EntityRepairImplTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 87 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 ++
...cEntitySerializationStrategyProxyV2Test.java | 83 ++
...ccEntitySerializationStrategyV1ImplTest.java | 55 +
...ccEntitySerializationStrategyV2ImplTest.java | 55 +
.../MvccEntitySerializationStrategyV2Test.java | 229 +++
.../impl/MvccLESSTransientTest.java | 55 +-
...ccLogEntrySerializationStrategyImplTest.java | 2 +-
.../impl/SettingsValidationTest.java | 131 ++
.../collection/util/EntityHelper.java | 105 ++
.../src/test/resources/log4j.properties | 21 +-
.../persistence/core/astyanax/CassandraFig.java | 12 +
.../persistence/core/astyanax/FieldBuffer.java | 70 +
.../core/astyanax/FieldBufferBuilder.java | 87 ++
.../core/astyanax/FieldBufferParser.java | 81 ++
.../core/astyanax/FieldBufferSerializer.java | 134 ++
.../core/guicyfig/SetConfigTestBypass.java | 108 ++
.../astyanax/FieldBufferSerializerTest.java | 126 ++
.../core/guice/MaxMigrationModule.java | 39 +
.../core/guice/MaxMigrationVersion.java | 40 +
.../core/guice/MigrationManagerRule.java | 19 +-
.../persistence/graph/GraphManagerIT.java | 2 +-
.../persistence/graph/GraphManagerLoadTest.java | 2 +-
.../graph/GraphManagerShardingIT.java | 2 +-
.../graph/GraphManagerStressTest.java | 2 +-
.../usergrid/persistence/graph/SimpleTest.java | 2 +-
.../graph/guice/TestGraphModule.java | 11 +
.../graph/impl/EdgeDeleteListenerTest.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 2 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 2 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 2 +-
.../EdgeMetadataSerializationTest.java | 2 +-
.../EdgeSerializationChopTest.java | 2 +-
.../serialization/EdgeSerializationTest.java | 2 +-
.../serialization/NodeSerializationTest.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 2 +-
.../NodeShardCounterSerializationTest.java | 2 +-
.../persistence/map/MapManagerTest.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 60 +-
.../index/impl/EsEntityIndexImpl.java | 60 +-
.../persistence/index/impl/EsProvider.java | 55 +-
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../index/impl/FailureMonitorImpl.java | 4 +-
.../persistence/index/impl/IndexingUtils.java | 47 +-
.../index/query/CandidateResults.java | 3 -
.../persistence/index/query/EntityResults.java | 4 +-
.../persistence/index/query/Identifier.java | 3 +-
.../usergrid/persistence/index/query/Query.java | 26 +-
.../persistence/queue/QueueManagerTest.java | 26 +-
.../apache/usergrid/rest/AbstractRestIT.java | 5 +-
.../collection/BrowserCompatibilityTest.java | 2 +-
.../collection/DuplicateNameIT.java | 2 +-
.../collection/activities/PutTest.java | 2 +-
.../collection/groups/GroupResourceIT.java | 240 +++-
.../collection/paging/PagingEntitiesTest.java | 4 +-
.../collection/paging/PagingResourceIT.java | 14 +-
.../users/ConnectionResourceTest.java | 16 +-
.../collection/users/OwnershipResourceIT.java | 18 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../applications/queries/AndOrQueryTest.java | 13 +-
.../queries/BadGrammarQueryTest.java | 2 +-
.../applications/queries/GeoPagingTest.java | 222 ++-
.../applications/queries/MatrixQueryTests.java | 2 +-
.../rest/applications/queries/OrderByTest.java | 6 +-
.../usergrid/rest/test/resource/Connection.java | 9 +
.../rest/test/resource/TestContext.java | 66 +-
.../rest/test/resource/ValueResource.java | 2 +
.../rest/test/resource/app/Application.java | 16 +-
.../rest/test/resource/app/Collection.java | 81 ++
.../usergrid/rest/test/resource/app/Group.java | 56 +
.../test/resource/app/GroupsCollection.java | 66 +
.../usergrid/rest/test/resource/app/Role.java | 56 +
.../rest/test/resource/app/RolesCollection.java | 64 +
.../rest/test/resource/app/UsersCollection.java | 26 +-
124 files changed, 6175 insertions(+), 1211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8555b1c,22aa05c..5aa9123
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -155,11 -145,16 +154,12 @@@ public class EsEntityIndexImpl implemen
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- Settings settings = ImmutableSettings.settingsBuilder()
- .put( "index.number_of_shards", numberOfShards )
- .put( "index.number_of_replicas", numberOfReplicas ).build();
+ Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
+ .put( "index.number_of_replicas", numberOfReplicas ).build();
+ createIndexAndAlias(admin, settings);
- final CreateIndexResponse cir = admin.indices().prepareCreate( indexName )
- .setSettings( settings ).execute().actionGet();
-
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+
// create the document, this ensures the index is ready
// Immediately create a document and remove it to ensure the entire cluster is ready
@@@ -176,15 -171,6 +176,15 @@@
}
}
+ private void createIndexAndAlias(AdminClient admin, Settings settings) {
- String indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), applicationScope);
++ String indexName = IndexingUtils.createIndexBaseName(config.getIndexPrefix(), applicationScope);
+ String indexVersionName = IndexingUtils.createIndexName(indexName, 0);
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
+ //check if alias exists and get the alias
+ admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
++ logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@@ -200,18 -186,20 +200,19 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
+ esProvider.getClient().prepareIndex( aliasName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+ .get();
- logger.info( "Successfully created new document with docId {} in index {} type {}",
- tempId, indexName, VERIFY_TYPE );
+ logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
+ aliasName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName )
- .setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+ esProvider.getClient().prepareDeleteByQuery( aliasName ).setTypes( VERIFY_TYPE )
+ .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+
+ logger.info( "Successfully deleted all documents in index {} and type {}", aliasName, VERIFY_TYPE );
- logger.info( "Successfully deleted all documents in index {} and type {}",
- indexName, VERIFY_TYPE );
+
return true;
}
};
@@@ -316,9 -302,8 +316,10 @@@
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
- new Object[] { this.indexName, context, entityTypes, srb } );
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
+ this.aliasName, context, entityTypes, srb
+ } );
++
}
@@@ -406,8 -391,9 +407,9 @@@
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName )
- .execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
+ esProvider.getClient().admin().indices().prepareRefresh( aliasName ).execute().actionGet();
+ logger.debug( "Refreshed index: " + aliasName );
++
return true;
}
catch ( IndexMissingException e ) {
@@@ -440,14 -427,13 +443,15 @@@
final String context = IndexingUtils.createContextName( scope );
final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery(
- IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+
+
+ final SearchRequestBuilder srb =
+ esProvider.getClient().prepareSearch( aliasName ).setTypes( searchTypes.getTypeNames() )
+ .setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
+
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( searchTypes.getTypeNames() ).setScroll( cursorTimeout + "m" )
- .setQuery( queryBuilder );
+
final SearchResponse searchResponse;
try {
searchResponse = srb.execute().actionGet();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 3aa4f39,6d34b15..315a043
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@@ -165,65 -151,69 +162,71 @@@ public class IndexingUtils
.startObject( type )
- .startArray( "dynamic_templates" )
+ .startArray("dynamic_templates")
- //we need most specific mappings first since it's a stop on match algorithm
+ // we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" )
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
-
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" ).endObject()
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed").endObject()
.endObject()
.endObject()
// any string with field name that starts with sa_ gets analyzed
.startObject()
- .startObject("template_1").field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field("match_mapping_type", "string").startObject( "mapping" ).field( "type", "string" )
- .field("index", "analyzed").endObject().endObject()
++
+ .startObject( "template_1" )
+ .field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "analyzed" )
+ .endObject()
.endObject()
+ .endObject()
+
+ // all other strings are not analyzed
+ .startObject()
+ .startObject( "template_2" )
+ //todo, should be string prefix, remove 2 field mapping
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
+
- // all other strings are not analyzed
- .startObject()
- .startObject("template_2")
- //todo, should be string prefix, remove 2 field mapping
- .field("match", "*").field( "match_mapping_type", "string" ).startObject( "mapping" )
- .field("type", "string")
- .field("index", "not_analyzed")
- .endObject()
.endObject()
.endObject()
+ .endObject()
- // fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
+ // fields names starting with go_ get geo-indexed
+ .startObject()
+ .startObject("template_3")
+ .field("match", GEO_PREFIX + "location")
+ .startObject("mapping")
+ .field("type", "geo_point")
++
.endObject()
- .endObject()
.endObject()
+ .endObject()
+ .endArray()
+ .endObject()
- .endArray()
-
- .endObject()
-
- .endObject();
+ .endObject();
return builder;
}