You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/12/03 23:29:08 UTC
[01/22] incubator-usergrid git commit: add alias support
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o da054a198 -> 7645a7910
add alias support
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7b08ea59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7b08ea59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7b08ea59
Branch: refs/heads/two-dot-o
Commit: 7b08ea599062ad550dfb0d9b4d05fdc0d2e8007a
Parents: 00eb339
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 13:43:36 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 13:43:36 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 6 ++
.../index/impl/EsEntityIndexBatchImpl.java | 8 +--
.../index/impl/EsEntityIndexImpl.java | 64 ++++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 8 +++
4 files changed, 57 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 0f405f3..c64ddff 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -37,6 +37,8 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
+ public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch_index_postfix";
+
public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
public static final String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
@@ -70,6 +72,10 @@ public interface IndexFig extends GuicyFig {
@Default( "usergrid" ) // no underbars allowed
@Key( ELASTICSEARCH_INDEX_PREFIX )
String getIndexPrefix();
+
+ @Default( "alias" ) // no underbars allowed
+ @Key( ELASTICSEARCH_ALIAS_POSTFIX )
+ String getAliasPostfix();
@Default( "1" ) // TODO: does this timeout get extended on each query?
@Key( QUERY_CURSOR_TIMEOUT_MINUTES )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index c4cdeeb..9f50ccd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -81,7 +81,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final boolean refresh;
- private final String indexName;
+ private final String aliasName;
private BulkRequestBuilder bulkRequest;
@@ -98,7 +98,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
- this.indexName = createIndexName( config.getIndexPrefix(), applicationScope );
+ this.aliasName = IndexingUtils.createAliasName(createIndexName( config.getIndexPrefix(), applicationScope ),config);
this.refresh = config.isForcedRefresh();
this.autoFlushSize = autoFlushSize;
initBatch();
@@ -137,7 +137,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource( entityAsMap ) );
+ bulkRequest.add( client.prepareIndex( aliasName, entityType, indexId ).setSource( entityAsMap ) );
maybeFlush();
@@ -169,7 +169,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete( aliasName, entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b1e5374..7407e49 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -28,13 +29,21 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.hppc.ObjectContainer;
+import org.elasticsearch.common.hppc.cursors.ObjectCursor;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -89,13 +98,13 @@ public class EsEntityIndexImpl implements EntityIndex {
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
+ private final String aliasName;
+
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
*/
private FailureMonitor failureMonitor;
- private final String indexName;
-
private final ApplicationScope applicationScope;
private final EsProvider esProvider;
@@ -128,7 +137,7 @@ public class EsEntityIndexImpl implements EntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexName = IndexingUtils.createIndexName( config.getIndexPrefix(), appScope );
+ this.aliasName = IndexingUtils.createAliasName(IndexingUtils.createIndexName(config.getIndexPrefix(), appScope),config);
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
@@ -146,14 +155,10 @@ public class EsEntityIndexImpl implements EntityIndex {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards )
+ Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
.put( "index.number_of_replicas", numberOfReplicas ).build();
-
- final CreateIndexResponse cir =
- admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
-
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ createIndexAndAlias(admin, settings);
// create the document, this ensures the index is ready
@@ -171,6 +176,15 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
+ private void createIndexAndAlias(AdminClient admin, Settings settings) {
+ String indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), applicationScope);
+ String indexVersionName = IndexingUtils.getIndexName(indexName, 0);
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
+ //check if alias exists and get the alias
+ admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@ -179,24 +193,24 @@ public class EsEntityIndexImpl implements EntityIndex {
private void testNewIndex() {
- logger.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", aliasName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+ esProvider.getClient().prepareIndex( aliasName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
.get();
logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- indexName, VERIFY_TYPE );
+ aliasName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
+ esProvider.getClient().prepareDeleteByQuery( aliasName ).setTypes( VERIFY_TYPE )
.setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}", aliasName, VERIFY_TYPE );
return true;
}
@@ -247,7 +261,7 @@ public class EsEntityIndexImpl implements EntityIndex {
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes )
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( aliasName ).setTypes( entityTypes )
.setScroll( cursorTimeout + "m" ).setQuery( qb );
@@ -303,7 +317,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
- this.indexName, context, entityTypes, srb
+ this.aliasName, context, entityTypes, srb
} );
}
@@ -386,14 +400,14 @@ public class EsEntityIndexImpl implements EntityIndex {
public void refresh() {
- logger.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", aliasName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName ).execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
+ esProvider.getClient().admin().indices().prepareRefresh( aliasName ).execute().actionGet();
+ logger.debug( "Refreshed index: " + aliasName );
return true;
}
catch ( IndexMissingException e ) {
@@ -405,7 +419,7 @@ public class EsEntityIndexImpl implements EntityIndex {
doInRetry( retryOperation );
- logger.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + aliasName );
}
@@ -430,7 +444,7 @@ public class EsEntityIndexImpl implements EntityIndex {
final SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( indexName ).setTypes( searchTypes.getTypeNames() )
+ esProvider.getClient().prepareSearch( aliasName ).setTypes( searchTypes.getTypeNames() )
.setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
@@ -456,12 +470,12 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( aliasName ).get();
if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + aliasName );
}
else {
- logger.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + aliasName );
}
}
@@ -520,7 +534,7 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health( new ClusterHealthRequest( new String[] { indexName } ) )
+ .health( new ClusterHealthRequest( new String[] { aliasName } ) )
.get();
return Health.valueOf( chr.getStatus().name() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c7592bc..dcd496b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -217,4 +218,11 @@ public class IndexingUtils {
return builder;
}
+ public static String createAliasName(String indexName, IndexFig fig) {
+ return indexName + fig.getAliasPostfix();
+ }
+
+ public static String getIndexName(String indexName, int i) {
+ return indexName+"_v"+(i+1);
+ }
}
[15/22] incubator-usergrid git commit: add index test
Posted by to...@apache.org.
add index test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/360a78e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/360a78e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/360a78e4
Branch: refs/heads/two-dot-o
Commit: 360a78e40b25475eb89d12a1b6ea5235f722e7e5
Parents: 72c165e
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Nov 26 21:28:15 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Nov 26 21:28:15 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/IndexResourceIT.java | 71 ++++++++++++++++++++
1 file changed, 71 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/360a78e4/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
new file mode 100644
index 0000000..bc33baa
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
@@ -0,0 +1,71 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.rest;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.junit.Assert.assertNull;
+
+/**
+ * Classy class class.
+ */
+public class IndexResourceIT extends AbstractRestIT {
+
+
+ @Rule
+ public TestContextSetup context = new TestContextSetup( this );
+ //Used for all MUUserResourceITTests
+ private Logger LOG = LoggerFactory.getLogger(IndexResourceIT.class);
+
+ public IndexResourceIT(){
+
+ }
+
+ @Test
+ public void TestAddIndex() throws Exception{
+ String superToken = superAdminToken();
+
+ Map<String, Object> data = new HashMap<String, Object>();
+ data.put( "replicas", 0 );
+ data.put( "shards", 1 );
+
+ UUID appId = this.context.getAppUuid();
+
+ // change the password as admin. The old password isn't required
+ JsonNode node = mapper.readTree( resource().path( "/index/"+appId ).queryParam( "access_token", superToken )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ .post( String.class, data ));
+
+ assertNull( getError( node ) );
+
+ refreshIndex("test-organization", "test-app");
+
+ }
+}
[03/22] incubator-usergrid git commit: merge from two-o
Posted by to...@apache.org.
merge from two-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3ff891e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3ff891e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3ff891e1
Branch: refs/heads/two-dot-o
Commit: 3ff891e159f2caf61a356c1f0fced387ac17595a
Parents: 55110a5 0a92bee
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 14:43:38 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 14:43:38 2014 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 32 +-
.../corepersistence/CpEntityDeleteListener.java | 9 +-
.../corepersistence/CpEntityManager.java | 44 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpRelationManager.java | 35 +-
.../usergrid/corepersistence/GuiceModule.java | 2 +
.../migration/EntityDataMigration.java | 147 ++
.../migration/EntityTypeMappingMigration.java | 28 +-
.../migration/GraphShardVersionMigration.java | 104 +-
.../corepersistence/migration/Versions.java | 3 +
.../rx/AllEntitiesInSystemObservable.java | 27 +-
.../rx/ApplicationObservable.java | 65 +-
.../corepersistence/util/CpNamingUtils.java | 17 +
.../org/apache/usergrid/CoreITSetupImpl.java | 13 +
.../CpEntityDeleteListenerTest.java | 4 +-
.../corepersistence/StaleIndexCleanupTest.java | 5 +-
.../migration/EntityDataMigrationIT.java | 251 ++++
.../migration/EntityTypeMappingMigrationIT.java | 74 +-
.../migration/GraphShardVersionMigrationIT.java | 153 +-
.../rx/AllEntitiesInSystemObservableIT.java | 23 +-
.../org/apache/usergrid/persistence/GeoIT.java | 404 ++++--
.../usergrid/persistence/LargeEntityIT.java | 112 ++
stack/core/src/test/resources/largeentity.json | 1329 ++++++++++++++++++
.../exception/EntityTooLargeException.java | 67 +
.../impl/EntityCollectionManagerImpl.java | 251 ++--
.../mvcc/MvccEntitySerializationStrategy.java | 23 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 3 +-
.../mvcc/stage/delete/MarkCommit.java | 5 +-
.../mvcc/stage/write/WriteCommit.java | 3 +-
.../serialization/SerializationFig.java | 15 +-
.../UniqueValueSerializationStrategy.java | 3 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 343 ++---
...vccEntitySerializationStrategyProxyImpl.java | 162 +++
.../MvccEntitySerializationStrategyV1Impl.java | 219 +++
.../MvccEntitySerializationStrategyV2Impl.java | 238 ++++
.../MvccLogEntrySerializationStrategyImpl.java | 2 +-
.../serialization/impl/SerializationModule.java | 25 +-
.../serialization/impl/SettingsValidation.java | 58 +
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../EntityCollectionManagerFactoryTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 64 +-
.../EntityCollectionManagerStressTest.java | 2 +-
.../EntityCollectionManagerSyncIT.java | 2 +-
.../collection/guice/MigrationManagerRule.java | 38 -
.../collection/guice/TestCollectionModule.java | 14 +
...niqueValueSerializationStrategyImplTest.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../serialization/EntityRepairImplTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 87 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 ++
...cEntitySerializationStrategyProxyV2Test.java | 83 ++
...ccEntitySerializationStrategyV1ImplTest.java | 55 +
...ccEntitySerializationStrategyV2ImplTest.java | 55 +
.../MvccEntitySerializationStrategyV2Test.java | 229 +++
.../impl/MvccLESSTransientTest.java | 55 +-
...ccLogEntrySerializationStrategyImplTest.java | 2 +-
.../impl/SettingsValidationTest.java | 131 ++
.../collection/util/EntityHelper.java | 105 ++
.../src/test/resources/log4j.properties | 21 +-
.../persistence/core/astyanax/CassandraFig.java | 12 +
.../persistence/core/astyanax/FieldBuffer.java | 70 +
.../core/astyanax/FieldBufferBuilder.java | 87 ++
.../core/astyanax/FieldBufferParser.java | 81 ++
.../core/astyanax/FieldBufferSerializer.java | 134 ++
.../core/guicyfig/SetConfigTestBypass.java | 108 ++
.../astyanax/FieldBufferSerializerTest.java | 126 ++
.../core/guice/MaxMigrationModule.java | 39 +
.../core/guice/MaxMigrationVersion.java | 40 +
.../core/guice/MigrationManagerRule.java | 19 +-
.../persistence/graph/GraphManagerIT.java | 2 +-
.../persistence/graph/GraphManagerLoadTest.java | 2 +-
.../graph/GraphManagerShardingIT.java | 2 +-
.../graph/GraphManagerStressTest.java | 2 +-
.../usergrid/persistence/graph/SimpleTest.java | 2 +-
.../graph/guice/TestGraphModule.java | 11 +
.../graph/impl/EdgeDeleteListenerTest.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 2 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 2 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 2 +-
.../EdgeMetadataSerializationTest.java | 2 +-
.../EdgeSerializationChopTest.java | 2 +-
.../serialization/EdgeSerializationTest.java | 2 +-
.../serialization/NodeSerializationTest.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 2 +-
.../NodeShardCounterSerializationTest.java | 2 +-
.../persistence/map/MapManagerTest.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 60 +-
.../index/impl/EsEntityIndexImpl.java | 60 +-
.../persistence/index/impl/EsProvider.java | 55 +-
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../index/impl/FailureMonitorImpl.java | 4 +-
.../persistence/index/impl/IndexingUtils.java | 47 +-
.../index/query/CandidateResults.java | 3 -
.../persistence/index/query/EntityResults.java | 4 +-
.../persistence/index/query/Identifier.java | 3 +-
.../usergrid/persistence/index/query/Query.java | 26 +-
.../persistence/queue/QueueManagerTest.java | 26 +-
.../apache/usergrid/rest/AbstractRestIT.java | 5 +-
.../collection/BrowserCompatibilityTest.java | 2 +-
.../collection/DuplicateNameIT.java | 2 +-
.../collection/activities/PutTest.java | 2 +-
.../collection/groups/GroupResourceIT.java | 240 +++-
.../collection/paging/PagingEntitiesTest.java | 4 +-
.../collection/paging/PagingResourceIT.java | 14 +-
.../users/ConnectionResourceTest.java | 16 +-
.../collection/users/OwnershipResourceIT.java | 18 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../applications/queries/AndOrQueryTest.java | 13 +-
.../queries/BadGrammarQueryTest.java | 2 +-
.../applications/queries/GeoPagingTest.java | 222 ++-
.../applications/queries/MatrixQueryTests.java | 2 +-
.../rest/applications/queries/OrderByTest.java | 6 +-
.../usergrid/rest/test/resource/Connection.java | 9 +
.../rest/test/resource/TestContext.java | 66 +-
.../rest/test/resource/ValueResource.java | 2 +
.../rest/test/resource/app/Application.java | 16 +-
.../rest/test/resource/app/Collection.java | 81 ++
.../usergrid/rest/test/resource/app/Group.java | 56 +
.../test/resource/app/GroupsCollection.java | 66 +
.../usergrid/rest/test/resource/app/Role.java | 56 +
.../rest/test/resource/app/RolesCollection.java | 64 +
.../rest/test/resource/app/UsersCollection.java | 26 +-
124 files changed, 6175 insertions(+), 1211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8555b1c,22aa05c..5aa9123
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -155,11 -145,16 +154,12 @@@ public class EsEntityIndexImpl implemen
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- Settings settings = ImmutableSettings.settingsBuilder()
- .put( "index.number_of_shards", numberOfShards )
- .put( "index.number_of_replicas", numberOfReplicas ).build();
+ Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
+ .put( "index.number_of_replicas", numberOfReplicas ).build();
+ createIndexAndAlias(admin, settings);
- final CreateIndexResponse cir = admin.indices().prepareCreate( indexName )
- .setSettings( settings ).execute().actionGet();
-
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+
// create the document, this ensures the index is ready
// Immediately create a document and remove it to ensure the entire cluster is ready
@@@ -176,15 -171,6 +176,15 @@@
}
}
+ private void createIndexAndAlias(AdminClient admin, Settings settings) {
- String indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), applicationScope);
++ String indexName = IndexingUtils.createIndexBaseName(config.getIndexPrefix(), applicationScope);
+ String indexVersionName = IndexingUtils.createIndexName(indexName, 0);
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
+ //check if alias exists and get the alias
+ admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
++ logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@@ -200,18 -186,20 +200,19 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
+ esProvider.getClient().prepareIndex( aliasName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+ .get();
- logger.info( "Successfully created new document with docId {} in index {} type {}",
- tempId, indexName, VERIFY_TYPE );
+ logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
+ aliasName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName )
- .setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+ esProvider.getClient().prepareDeleteByQuery( aliasName ).setTypes( VERIFY_TYPE )
+ .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+
+ logger.info( "Successfully deleted all documents in index {} and type {}", aliasName, VERIFY_TYPE );
- logger.info( "Successfully deleted all documents in index {} and type {}",
- indexName, VERIFY_TYPE );
+
return true;
}
};
@@@ -316,9 -302,8 +316,10 @@@
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
- new Object[] { this.indexName, context, entityTypes, srb } );
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
+ this.aliasName, context, entityTypes, srb
+ } );
++
}
@@@ -406,8 -391,9 +407,9 @@@
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName )
- .execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
+ esProvider.getClient().admin().indices().prepareRefresh( aliasName ).execute().actionGet();
+ logger.debug( "Refreshed index: " + aliasName );
++
return true;
}
catch ( IndexMissingException e ) {
@@@ -440,14 -427,13 +443,15 @@@
final String context = IndexingUtils.createContextName( scope );
final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery(
- IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+
+
+ final SearchRequestBuilder srb =
+ esProvider.getClient().prepareSearch( aliasName ).setTypes( searchTypes.getTypeNames() )
+ .setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
+
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( searchTypes.getTypeNames() ).setScroll( cursorTimeout + "m" )
- .setQuery( queryBuilder );
+
final SearchResponse searchResponse;
try {
searchResponse = srb.execute().actionGet();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ff891e1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 3aa4f39,6d34b15..315a043
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@@ -165,65 -151,69 +162,71 @@@ public class IndexingUtils
.startObject( type )
- .startArray( "dynamic_templates" )
+ .startArray("dynamic_templates")
- //we need most specific mappings first since it's a stop on match algorithm
+ // we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" )
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
-
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" ).endObject()
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed").endObject()
.endObject()
.endObject()
// any string with field name that starts with sa_ gets analyzed
.startObject()
- .startObject("template_1").field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field("match_mapping_type", "string").startObject( "mapping" ).field( "type", "string" )
- .field("index", "analyzed").endObject().endObject()
++
+ .startObject( "template_1" )
+ .field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "analyzed" )
+ .endObject()
.endObject()
+ .endObject()
+
+ // all other strings are not analyzed
+ .startObject()
+ .startObject( "template_2" )
+ //todo, should be string prefix, remove 2 field mapping
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
+
- // all other strings are not analyzed
- .startObject()
- .startObject("template_2")
- //todo, should be string prefix, remove 2 field mapping
- .field("match", "*").field( "match_mapping_type", "string" ).startObject( "mapping" )
- .field("type", "string")
- .field("index", "not_analyzed")
- .endObject()
.endObject()
.endObject()
+ .endObject()
- // fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
+ // fields names starting with go_ get geo-indexed
+ .startObject()
+ .startObject("template_3")
+ .field("match", GEO_PREFIX + "location")
+ .startObject("mapping")
+ .field("type", "geo_point")
++
.endObject()
- .endObject()
.endObject()
+ .endObject()
+ .endArray()
+ .endObject()
- .endArray()
-
- .endObject()
-
- .endObject();
+ .endObject();
return builder;
}
[21/22] incubator-usergrid git commit: Merge branch 'two-dot-o' into
index-alias
Posted by to...@apache.org.
Merge branch 'two-dot-o' into index-alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e7455e9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e7455e9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e7455e9f
Branch: refs/heads/two-dot-o
Commit: e7455e9fdc0099ae39bc8b4b4ce9bdeb4e5d2b1b
Parents: 887bb47 c2bc921
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Dec 3 10:59:27 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Dec 3 10:59:27 2014 -0700
----------------------------------------------------------------------
stack/awscluster/gatling-cluster-cf.json | 387 +++++++++++++++++++
stack/awscluster/pom.xml | 2 +-
.../main/dist/init_instance/install_yourkit.sh | 6 +-
.../src/main/groovy/NodeRegistry.groovy | 197 +++++++---
.../main/groovy/configure_elasticsearch.groovy | 31 +-
.../src/main/groovy/registry_list.groovy | 36 --
stack/awscluster/ugcluster-cf.json | 7 +-
.../MvccEntitySerializationStrategyImpl.java | 2 +-
.../persistence/model/field/LocationField.java | 8 +
stack/loadtests/README.md | 19 +
stack/loadtests/pom.xml | 4 +-
.../usergrid/scenarios/DeviceScenarios.scala | 2 +-
.../usergrid/scenarios/UserScenarios.scala | 54 ++-
.../org/apache/usergrid/settings/Settings.scala | 2 +-
.../simulations/PostUsersSimulation.scala | 49 +--
.../usergrid/simulations/SetupSimulation.scala | 4 +-
stack/loadtests/src/test/scala/Engine.scala | 4 +-
stack/pom.xml | 1 +
18 files changed, 654 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
[11/22] incubator-usergrid git commit: add another test for index
Posted by to...@apache.org.
add another test for index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0273c558
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0273c558
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0273c558
Branch: refs/heads/two-dot-o
Commit: 0273c5581f827dfa586bc0ddaa18d783c37f1a18
Parents: b376cfd
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Nov 25 14:18:51 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Nov 25 14:18:51 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EntityIndexTest.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0273c558/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 8ea8835..0e0e81d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -131,8 +131,12 @@ public class EntityIndexTest extends BaseIT {
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
- testQueries( indexScope, searchTypes, entityIndex );
+ entityIndex.refresh();
+
+ //Hilda Youn
+ testQuery(indexScope, searchTypes, entityIndex, "name = 'Hilda Young'", 1 );
+ testQuery(indexScope, searchTypes, entityIndex, "name = 'Lowe Kelley'", 1 );
}
private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope, String filePath,final int max,final int startIndex) throws IOException {
[12/22] incubator-usergrid git commit: adding config based interface
so nothing is es specific, adding rest routes
Posted by to...@apache.org.
adding config based interface so nothing is es specific, adding rest routes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cfa73b43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cfa73b43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cfa73b43
Branch: refs/heads/two-dot-o
Commit: cfa73b43a896639f5c28653b6f60d6b550549cc5
Parents: 0273c55
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Nov 25 17:37:37 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Nov 25 17:37:37 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 5 +
.../HybridEntityManagerFactory.java | 5 +
.../persistence/EntityManagerFactory.java | 9 +
.../cassandra/EntityManagerFactoryImpl.java | 5 +
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../index/impl/EsEntityIndexImpl.java | 13 +-
.../persistence/index/impl/EntityIndexTest.java | 6 +-
.../org/apache/usergrid/rest/IndexResource.java | 215 +++++++++++++++++++
.../apache/usergrid/rest/SystemResource.java | 151 +------------
9 files changed, 262 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 82e80a5..5a09423 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -701,6 +701,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
+ public void addIndex(final UUID applicationId,final String indexSuffix,final Map<String,Object> config){
+ getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope( applicationId )).addIndex(indexSuffix,config);
+ }
+
+ @Override
public Health getEntityStoreHealth() {
// could use any collection scope here, does not matter
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 826d19d..0f1fbed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -198,6 +198,11 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
}
@Override
+ public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
+ factory.addIndex(appId,suffix,config);
+ }
+
+ @Override
public Health getEntityStoreHealth() {
return factory.getEntityStoreHealth();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 5dffb9f..33413a0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.springframework.context.ApplicationContext;
@@ -160,6 +161,14 @@ public interface EntityManagerFactory {
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
+ /**
+ * Add a new index to the application for scale
+ * @param appId application id
+ * @param suffix unique indentifier for additional index
+ * @param config {"replicas":num,"shards":num}
+ */
+ public void addIndex(final UUID appId,final String suffix, Map<String,Object> config);
+
public Health getEntityStoreHealth();
public interface ProgressObserver {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 8d9e239..68abf2e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -472,6 +472,11 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
+ public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
+ throw new UnsupportedOperationException("Not supported in v1");
+ }
+
+ @Override
public Health getEntityStoreHealth() {
throw new UnsupportedOperationException("Not supported yet.");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 30dc3fc..93dd3ea 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -24,6 +24,8 @@ import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.model.entity.Id;
+import java.util.Map;
+
/**
* Provides indexing of Entities within a scope.
@@ -39,8 +41,9 @@ public interface EntityIndex {
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
+ * @param config {"replicas":num,"shards":num}
*/
- public void createIndexAddToAlias(final String indexSuffix, final int numShards, final int numReplicas);
+ public void addIndex(final String indexSuffix, final Map<String,Object> config);
/**
* Create the index batch.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index e7cbd2e..623788e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -133,11 +134,19 @@ public class EsEntityIndexImpl implements EntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- createIndexAddToAlias(null,numberOfShards,numberOfReplicas);
+ addIndex(null, numberOfShards, numberOfReplicas);
}
@Override
- public void createIndexAddToAlias(final String indexSuffix, final int numberOfShards, final int numberOfReplicas) {
+ public void addIndex(final String indexSuffix, final Map<String,Object> config) {
+ if(!config.containsKey("replicas") || !config.containsKey("shards")){
+ throw new IllegalArgumentException("config must contains 'replicas' and 'shards'");
+ }
+ addIndex(indexSuffix, (int) config.get("shards"),(int)config.get("replicas"));
+ }
+
+ private void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
+
try {
if (!mappingsCreated.getAndSet(true)) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 0e0e81d..360347a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -127,7 +127,11 @@ public class EntityIndexTest extends BaseIT {
testQueries( indexScope, searchTypes, entityIndex );
- entityIndex.createIndexAddToAlias("v2",1,0);
+ HashMap<String,Object> config = new HashMap<>();
+ config.put("replicas",0);
+ config.put("shards",1);
+
+ entityIndex.addIndex("v2", config);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
new file mode 100644
index 0000000..f95d439
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -0,0 +1,215 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.rest;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(IndexResource.class);
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+
+ final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+
+
+ @Override
+ public void onProgress( final EntityRef entity ) {
+ logger.info( "Indexing entity {}:{} ", entity.getType(), entity.getUuid() );
+ }
+
+
+ @Override
+ public long getWriteDelayTime() {
+ return 0;
+ }
+ };
+
+
+ final Thread rebuild = new Thread() {
+
+ @Override
+ public void run() {
+ logger.info( "Rebuilding all indexes" );
+
+ try {
+ emf.rebuildAllIndexes( po );
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to rebuild indexes", e );
+ }
+ }
+ };
+
+ rebuild.setName( "Index rebuild all usergrid" );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+
+ final EntityManager em = emf.getEntityManager( appId );
+
+ final Set<String> collectionNames = em.getApplicationCollections();
+
+ final Thread rebuild = new Thread() {
+
+ @Override
+ public void run() {
+ for ( String collectionName : collectionNames )
+
+
+ {
+ rebuildCollection( appId, collectionName, delay );
+ }
+ }
+ };
+
+ rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+ throws Exception {
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+ final Thread rebuild = new Thread() {
+
+ public void run() {
+
+ rebuildCollection( appId, collectionName, delay );
+ }
+ };
+
+ rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ public JSONWithPadding addIndex(@Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "indexSuffix" ) final String indexSuffix,
+ @PathParam("shards") final int shards,
+ @PathParam("replicas") final int replicas,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) {
+ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
+ emf.addIndex(appId, indexSuffix, config);
+ ApiResponse response = createApiResponse();
+ response.setAction("Add index to alias");
+ return new JSONWithPadding(response, callback);
+ }
+
+ private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {
+ EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+
+ @Override
+ public void onProgress( final EntityRef entity ) {
+ logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
+ }
+ @Override
+ public long getWriteDelayTime() {
+ return delay;
+ }
+ };
+
+
+ logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
+
+ emf.rebuildCollectionIndex( applicationId, collectionName, po );
+ emf.refreshIndex();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index 1961aa1..fca3333 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -117,155 +117,10 @@ public class SystemResource extends AbstractContextResource {
}
@Path( "migrate" )
- public MigrateResource doSomething(){
+ public MigrateResource migrate(){
return getSubResource( MigrateResource.class );
}
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
-
- final ProgressObserver po = new ProgressObserver() {
-
-
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{} ", entity.getType(), entity.getUuid() );
- }
-
-
- @Override
- public long getWriteDelayTime() {
- return 0;
- }
- };
-
-
- final Thread rebuild = new Thread() {
-
- @Override
- public void run() {
- logger.info( "Rebuilding all indexes" );
-
- try {
- emf.rebuildAllIndexes( po );
- }
- catch ( Exception e ) {
- logger.error( "Unable to rebuild indexes", e );
- }
- }
- };
-
- rebuild.setName( "Index rebuild all usergrid" );
- rebuild.setDaemon( true );
- rebuild.start();
-
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
-
- final EntityManager em = emf.getEntityManager( appId );
-
- final Set<String> collectionNames = em.getApplicationCollections();
-
- final Thread rebuild = new Thread() {
-
- @Override
- public void run() {
- for ( String collectionName : collectionNames )
-
-
- {
- rebuildCollection( appId, collectionName, delay );
- }
- }
- };
-
- rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
- rebuild.setDaemon( true );
- rebuild.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
- throws Exception {
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
- final Thread rebuild = new Thread() {
-
- public void run() {
-
- rebuildCollection( appId, collectionName, delay );
- }
- };
-
- rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
- rebuild.setDaemon( true );
- rebuild.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
-
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
- }
-
-
- @Override
- public long getWriteDelayTime() {
- return delay;
- }
- };
-
-
- logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
-
- emf.rebuildCollectionIndex( applicationId, collectionName, po );
- emf.refreshIndex();
- }
+ @Path( "index" )
+ public IndexResource index() { return getSubResource(IndexResource.class); }
}
[05/22] incubator-usergrid git commit: adding comments
Posted by to...@apache.org.
adding comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5d613dca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5d613dca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5d613dca
Branch: refs/heads/two-dot-o
Commit: 5d613dca337e23314851b4c0e1aba3e5fa3788e9
Parents: b16c728
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 16:42:36 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 16:42:36 2014 -0700
----------------------------------------------------------------------
.../persistence/index/IndexIdentifier.java | 21 ++++++++++++++++----
.../index/impl/EsEntityIndexImpl.java | 5 +++++
2 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d613dca/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 13af8eb..c65a740 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -35,19 +35,32 @@ public class IndexIdentifier{
this.applicationScope = applicationScope;
}
+ /**
+ * Get the alias name
+ * @return
+ */
public String getAlias() {
return getIndexBase() + config.getAliasPostfix();
}
- public String getIndex(int i) {
- if (i > 0) {
- return getIndexBase() + "_v" + (i + 1);
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param version
+ * @return
+ */
+ public String getIndex(int version) {
+ if (version > 0) {
+ return getIndexBase() + "_v" + (version + 1);
} else {
return getIndexBase();
}
}
- String getIndexBase() {
+ /**
+ * returns the base name for index which will be used to add an alias and index
+ * @return
+ */
+ private String getIndexBase() {
StringBuilder sb = new StringBuilder();
sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
IndexingUtils.idString(sb, applicationScope.getApplication());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d613dca/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index baa3551..fa5197e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -174,6 +174,11 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
+ /**
+ * Create the index and alias
+ * @param admin
+ * @param settings
+ */
private void createIndexAndAlias(AdminClient admin, Settings settings) {
String indexVersionName = indexIdentifier.getIndex(0);
final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
[22/22] incubator-usergrid git commit: Merge branch 'two-dot-o' into
index-alias
Posted by to...@apache.org.
Merge branch 'two-dot-o' into index-alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7645a791
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7645a791
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7645a791
Branch: refs/heads/two-dot-o
Commit: 7645a7910d3df782fe56636a6626c59f987a3bfd
Parents: e7455e9 da054a1
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Dec 3 13:27:00 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Dec 3 13:27:00 2014 -0700
----------------------------------------------------------------------
.../src/main/groovy/NodeRegistry.groovy | 12 +-
.../org/apache/usergrid/persistence/GeoIT.java | 579 ++++++++++---------
.../core/astyanax/BucketScopedRowKey.java | 3 +-
.../core/astyanax/CassandraConfig.java | 18 +
.../core/astyanax/ColumnNameIterator.java | 1 -
.../persistence/core/astyanax/ColumnSearch.java | 30 +-
.../persistence/core/astyanax/ColumnTypes.java | 19 +-
.../core/astyanax/FieldBufferBuilder.java | 1 -
.../astyanax/MultiKeyColumnNameIterator.java | 2 -
.../core/astyanax/MultiRowColumnIterator.java | 4 -
.../core/astyanax/MultiTennantColumnFamily.java | 1 -
.../persistence/core/astyanax/ScopedRowKey.java | 1 -
.../astyanax/StringRowCompositeSerializer.java | 35 +-
.../core/hystrix/HystrixCassandra.java | 3 -
.../core/migration/data/DataMigration.java | 31 +-
.../migration/data/DataMigrationException.java | 30 +-
.../migration/data/DataMigrationManager.java | 30 +-
.../data/DataMigrationManagerImpl.java | 3 -
.../data/MigrationInfoSerialization.java | 31 +-
.../data/MigrationInfoSerializationImpl.java | 31 +-
.../migration/schema/MigrationManagerFig.java | 18 +
.../migration/schema/MigrationManagerImpl.java | 18 +
.../core/shard/ExpandingShardLocator.java | 23 +-
.../persistence/core/shard/StringHashUtils.java | 22 +-
.../usergrid/persistence/core/task/Task.java | 19 +-
.../persistence/core/task/TaskExecutor.java | 18 +
.../simulations/PostUsersSimulation.scala | 2 +-
27 files changed, 522 insertions(+), 463 deletions(-)
----------------------------------------------------------------------
[17/22] incubator-usergrid git commit: add more validation
Posted by to...@apache.org.
add more validation
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d28af0d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d28af0d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d28af0d8
Branch: refs/heads/two-dot-o
Commit: d28af0d8fa4fb97cdad51504f825d8cb3baec93a
Parents: e2c1a53
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 2 15:19:26 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 2 15:19:26 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d28af0d8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 588a5a1..0b1cb0c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -140,11 +140,12 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public void addIndex(final String indexSuffix, final Map<String,Object> config) {
- if(!config.containsKey("replicas") || !config.containsKey("shards")){
- throw new IllegalArgumentException("config must contains 'replicas' and 'shards'");
+ if(!config.containsKey("replicas") || !config.containsKey("shards") ||
+ !(config.get("replicas") instanceof Integer) || !(config.get("shards") instanceof Integer)){
+ throw new IllegalArgumentException("config must contains 'replicas' of type int and 'shards' of type int");
}
String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
- addIndex(normalizedSuffix, (int) config.get("shards"),(int)config.get("replicas"));
+ addIndex(normalizedSuffix,(int) config.get("shards"),(int)config.get("replicas"));
}
private void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
[09/22] incubator-usergrid git commit: add write_alias and read_alias
Posted by to...@apache.org.
add write_alias and read_alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/af16be29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/af16be29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/af16be29
Branch: refs/heads/two-dot-o
Commit: af16be295a880c3bb62dbc013c656380b5fe7537
Parents: 486c0d2
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Nov 24 16:55:33 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Nov 24 16:55:33 2014 -0700
----------------------------------------------------------------------
.../persistence/index/IndexIdentifier.java | 21 ++++-
.../index/impl/EsEntityIndexBatchImpl.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 88 +++++++++-----------
3 files changed, 64 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/af16be29/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 58e6ee5..82a6023 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -39,8 +39,8 @@ public class IndexIdentifier{
* Get the alias name
* @return
*/
- public String getAlias() {
- return getIndexBase() + "_" + config.getAliasPostfix();
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,getIndexBase());
}
/**
@@ -67,5 +67,22 @@ public class IndexIdentifier{
return sb.toString();
}
+ public class IndexAlias{
+ private final String readAlias;
+ private final String writeAlias;
+
+ public IndexAlias(IndexFig indexFig,String indexBase) {
+ this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix();
+ this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix();
+ }
+
+ public String getReadAlias() {
+ return readAlias;
+ }
+
+ public String getWriteAlias() {
+ return writeAlias;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/af16be29/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 9589e66..0c2dec0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -81,7 +81,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final boolean refresh;
- private final String aliasName;
+ private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
private BulkRequestBuilder bulkRequest;
@@ -100,7 +100,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.client = client;
this.failureMonitor = failureMonitor;
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
- this.aliasName = indexIdentifier.getAlias();
+ this.alias = indexIdentifier.getAlias();
this.refresh = config.isForcedRefresh();
this.autoFlushSize = autoFlushSize;
initBatch();
@@ -144,7 +144,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- bulkRequest.add( client.prepareIndex( aliasName, entityType, indexId ).setSource( entityAsMap ) );
+ bulkRequest.add( client.prepareIndex(alias.getWriteAlias(), entityType, indexId ).setSource( entityAsMap ) );
maybeFlush();
@@ -182,7 +182,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- bulkRequest.add( client.prepareDelete( aliasName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(alias.getReadAlias(), entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/af16be29/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f9623ce..540aec4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -30,21 +29,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.hppc.ObjectContainer;
-import org.elasticsearch.common.hppc.cursors.ObjectCursor;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -93,7 +84,7 @@ public class EsEntityIndexImpl implements EntityIndex {
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
- private final String aliasName;
+ private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
/**
@@ -133,8 +124,8 @@ public class EsEntityIndexImpl implements EntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexIdentifier = IndexingUtils.createIndexIdentifier(config,appScope);
- this.aliasName = indexIdentifier.getAlias();
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
+ this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
@@ -176,19 +167,22 @@ public class EsEntityIndexImpl implements EntityIndex {
Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
.put( "index.number_of_replicas", numberOfReplicas ).build();
+ //TODO:swallow exception, look into setting up routing rules
String indexVersionName = indexIdentifier.getIndex(0);
- if(!admin.indices().exists(new IndicesExistsRequest(indexVersionName)).actionGet().isExists()) {
+ try {
final CreateIndexResponse cir = admin.indices().prepareCreate(indexVersionName).setSettings(settings).execute().actionGet();
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
- }else{
- logger.info( " Index Name [{}] already exists", indexVersionName);
+ logger.info("Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged());
+ }catch(IndexAlreadyExistsException e){
+ logger.info("Index Name [{}] already exists",indexVersionName);
}
- //check if alias exists and get the alias
- if(!admin.indices().aliasesExist(new GetAliasesRequest(aliasName)).actionGet().exists()) {
- final Boolean isAck = admin.indices().prepareAliases().addAlias(indexVersionName, aliasName).execute().actionGet().isAcknowledged();
- logger.info( "Created new Alias Name [{}] ACK=[{}]", aliasName, isAck);
- }else{
- logger.info( " Alias Name [{}] already exists", aliasName);
+ try { //check if alias exists and get the alias
+ Boolean isAck = admin.indices().prepareAliases().addAlias(indexVersionName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+
+ isAck = admin.indices().prepareAliases().addAlias(indexVersionName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
+ } catch (Exception e) {
+ logger.warn("Failed to create alias ", e);
}
}
@@ -200,24 +194,24 @@ public class EsEntityIndexImpl implements EntityIndex {
private void testNewIndex() {
- logger.info( "Refreshing Created new Index Name [{}]", aliasName );
+ logger.info( "Refreshing Created new Index Name [{}]", alias);
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( aliasName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+ esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
.get();
logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- aliasName, VERIFY_TYPE );
+ alias, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( aliasName ).setTypes( VERIFY_TYPE )
+ esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setTypes(VERIFY_TYPE)
.setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- logger.info( "Successfully deleted all documents in index {} and type {}", aliasName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}", alias, VERIFY_TYPE );
return true;
@@ -234,8 +228,8 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
private void createMappings() throws IOException {
- XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_" );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
+ XContentFactory.jsonBuilder(), "_default_");
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
@@ -260,7 +254,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName( indexScope );
+ final String context = IndexingUtils.createContextName(indexScope);
final String[] entityTypes = searchTypes.getTypeNames();
QueryBuilder qb = query.createQueryBuilder( context );
@@ -269,8 +263,8 @@ public class EsEntityIndexImpl implements EntityIndex {
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( aliasName ).setTypes( entityTypes )
- .setScroll( cursorTimeout + "m" ).setQuery( qb );
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(entityTypes)
+ .setScroll(cursorTimeout + "m").setQuery(qb);
@@ -324,7 +318,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
- this.aliasName, context, entityTypes, srb
+ this.alias, context, entityTypes, srb
} );
}
@@ -368,7 +362,7 @@ public class EsEntityIndexImpl implements EntityIndex {
failureMonitor.success();
}
- return parseResults( searchResponse, query );
+ return parseResults(searchResponse, query);
}
@@ -398,7 +392,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( candidates.size() >= query.getLimit() ) {
candidateResults.setCursor( searchResponse.getScrollId() );
- logger.debug( " Cursor = " + searchResponse.getScrollId() );
+ logger.debug(" Cursor = " + searchResponse.getScrollId());
}
return candidateResults;
@@ -408,14 +402,14 @@ public class EsEntityIndexImpl implements EntityIndex {
public void refresh() {
- logger.info( "Refreshing Created new Index Name [{}]", aliasName );
+ logger.info( "Refreshing Created new Index Name [{}]", alias);
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( aliasName ).execute().actionGet();
- logger.debug( "Refreshed index: " + aliasName );
+ esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
+ logger.debug( "Refreshed index: " + alias);
return true;
}
@@ -428,7 +422,7 @@ public class EsEntityIndexImpl implements EntityIndex {
doInRetry( retryOperation );
- logger.debug( "Refreshed index: " + aliasName );
+ logger.debug( "Refreshed index: " + alias);
}
@@ -447,15 +441,15 @@ public class EsEntityIndexImpl implements EntityIndex {
//since we don't have paging inputs, there's no point in executing a query for paging.
- final String context = IndexingUtils.createContextName( scope );
+ final String context = IndexingUtils.createContextName(scope);
final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
final SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( aliasName ).setTypes( searchTypes.getTypeNames() )
- .setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
+ esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(searchTypes.getTypeNames())
+ .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
@@ -472,7 +466,7 @@ public class EsEntityIndexImpl implements EntityIndex {
failureMonitor.success();
- return parseResults( searchResponse, new Query( ) );
+ return parseResults(searchResponse, new Query());
}
@@ -481,12 +475,12 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( aliasName ).get();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexIdentifier.getIndex(0) ).get();
if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: " + aliasName );
+ logger.info( "Deleted index: " + alias);
}
else {
- logger.info( "Failed to delete index " + aliasName );
+ logger.info( "Failed to delete index " + alias);
}
}
@@ -545,7 +539,7 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health( new ClusterHealthRequest( new String[] { aliasName } ) )
+ .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(0)}))
.get();
return Health.valueOf( chr.getStatus().name() );
}
[06/22] incubator-usergrid git commit: check for alias existence
Posted by to...@apache.org.
check for alias existence
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fc796285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fc796285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fc796285
Branch: refs/heads/two-dot-o
Commit: fc7962855a8acb84231c7f9432f01614b15dbc6e
Parents: 5d613dc
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 16:58:11 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 16:58:11 2014 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 34 +++++++++++---------
1 file changed, 18 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fc796285/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index fa5197e..aab8301 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -147,16 +147,7 @@ public class EsEntityIndexImpl implements EntityIndex {
createMappings();
}
- final AdminClient admin = esProvider.getClient().admin();
-
- final int numberOfShards = config.getNumberOfShards();
- final int numberOfReplicas = config.getNumberOfReplicas();
-
- Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
- .put( "index.number_of_replicas", numberOfReplicas ).build();
-
- createIndexAndAlias(admin, settings);
-
+ createIndexAndAlias();
// create the document, this ensures the index is ready
@@ -176,15 +167,26 @@ public class EsEntityIndexImpl implements EntityIndex {
/**
* Create the index and alias
- * @param admin
- * @param settings
*/
- private void createIndexAndAlias(AdminClient admin, Settings settings) {
+ private void createIndexAndAlias() {
+ final int numberOfShards = config.getNumberOfShards();
+ final int numberOfReplicas = config.getNumberOfReplicas();
+ final AdminClient admin = esProvider.getClient().admin();
+
+ Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
+ .put( "index.number_of_replicas", numberOfReplicas ).build();
+
String indexVersionName = indexIdentifier.getIndex(0);
- final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
+ if(!admin.indices().exists(new IndicesExistsRequest(indexVersionName)).actionGet().isExists()) {
+ final CreateIndexResponse cir = admin.indices().prepareCreate(indexVersionName).setSettings(settings).execute().actionGet();
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
+ }
//check if alias exists and get the alias
- admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
+ if(!admin.indices().aliasesExist(new GetAliasesRequest(aliasName)).actionGet().exists()) {
+ final Boolean isAck = admin.indices().prepareAliases().addAlias(indexVersionName, aliasName).execute().actionGet().isAcknowledged();
+ logger.info( "Created new Alias Name [{}] ACK=[{}]", aliasName, isAck);
+
+ }
}
[04/22] incubator-usergrid git commit: add index naming util
Posted by to...@apache.org.
add index naming util
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b16c728f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b16c728f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b16c728f
Branch: refs/heads/two-dot-o
Commit: b16c728f0526c0ac7730e4c2c35cd47dd64fcd6b
Parents: 3ff891e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 15:47:52 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 15:47:52 2014 -0700
----------------------------------------------------------------------
.../persistence/index/IndexIdentifier.java | 58 ++++++++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 6 +-
.../index/impl/EsEntityIndexImpl.java | 19 +++----
.../persistence/index/impl/IndexingUtils.java | 25 +++------
4 files changed, 77 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b16c728f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
new file mode 100644
index 0000000..13af8eb
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -0,0 +1,58 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+
+/**
+ * Class is used to generate an index name and alias name
+ */
+public class IndexIdentifier{
+ private final IndexFig config;
+ private final ApplicationScope applicationScope;
+
+ public IndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
+ this.config = config;
+ this.applicationScope = applicationScope;
+ }
+
+ public String getAlias() {
+ return getIndexBase() + config.getAliasPostfix();
+ }
+
+ public String getIndex(int i) {
+ if (i > 0) {
+ return getIndexBase() + "_v" + (i + 1);
+ } else {
+ return getIndexBase();
+ }
+ }
+
+ String getIndexBase() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ IndexingUtils.idString(sb, applicationScope.getApplication());
+ return sb.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b16c728f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index da1e930..9589e66 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -67,7 +68,6 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFI
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
@@ -82,6 +82,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final boolean refresh;
private final String aliasName;
+ private final IndexIdentifier indexIdentifier;
private BulkRequestBuilder bulkRequest;
@@ -98,7 +99,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
- this.aliasName = IndexingUtils.createAliasName( config, applicationScope );
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
+ this.aliasName = indexIdentifier.getAlias();
this.refresh = config.isForcedRefresh();
this.autoFlushSize = autoFlushSize;
initBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b16c728f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 5aa9123..baa3551 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.usergrid.persistence.index.*;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@ -65,11 +66,6 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
@@ -98,6 +94,7 @@ public class EsEntityIndexImpl implements EntityIndex {
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
private final String aliasName;
+ private final IndexIdentifier indexIdentifier;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -136,7 +133,8 @@ public class EsEntityIndexImpl implements EntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.aliasName = IndexingUtils.createAliasName(config,appScope);
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier(config,appScope);
+ this.aliasName = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
@@ -177,8 +175,7 @@ public class EsEntityIndexImpl implements EntityIndex {
}
private void createIndexAndAlias(AdminClient admin, Settings settings) {
- String indexName = IndexingUtils.createIndexBaseName(config.getIndexPrefix(), applicationScope);
- String indexVersionName = IndexingUtils.createIndexName(indexName, 0);
+ String indexVersionName = indexIdentifier.getIndex(0);
final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
//check if alias exists and get the alias
admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
@@ -231,9 +228,9 @@ public class EsEntityIndexImpl implements EntityIndex {
XContentFactory.jsonBuilder(), "_default_" );
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
- .preparePutTemplate( "usergrid_template" )
+ .preparePutTemplate("usergrid_template")
// set mapping as the default for all types
- .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", xcb )
+ .setTemplate(config.getIndexPrefix() + "*").addMapping( "_default_", xcb )
.execute().actionGet();
if(!pitr.isAcknowledged()){
@@ -346,7 +343,7 @@ public class EsEntityIndexImpl implements EntityIndex {
logger.debug( "Executing query with cursor: {} ", scrollId );
SearchScrollRequestBuilder ssrb = esProvider.getClient()
- .prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
+ .prepareSearchScroll(scrollId).setScroll( cursorTimeout + "m" );
try {
searchResponse = ssrb.execute().actionGet();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b16c728f/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 315a043..b3ce7f8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -23,6 +23,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -90,30 +91,18 @@ public class IndexingUtils {
/**
- * Create the index name based on our prefix+appUUID+AppType
- * @param prefix
+ * Create the facilities to retrieve an index name and alias name
+ * @param fig
* @param applicationScope
* @return
*/
- public static String createIndexBaseName(String prefix, ApplicationScope applicationScope) {
- StringBuilder sb = new StringBuilder();
- sb.append( prefix ).append( SEPARATOR );
- idString( sb, applicationScope.getApplication() );
- return sb.toString();
+ public static IndexIdentifier createIndexIdentifier(IndexFig fig, ApplicationScope applicationScope) {
+ return new IndexIdentifier(fig,applicationScope);
}
- public static String createIndexName(String indexName, int i) {
- if(i>0) {
- return indexName + "_v" + (i + 1);
- }else{
- return indexName;
- }
- }
- public static String createAliasName( IndexFig config,ApplicationScope applicationScope) {
- String indexName =IndexingUtils.createIndexBaseName(config.getIndexPrefix(), applicationScope);
- return indexName + config.getAliasPostfix();
- }
+
+
/**
* Create the index doc from the given entity
[07/22] incubator-usergrid git commit: testing multiple index
creations
Posted by to...@apache.org.
testing multiple index creations
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/30b38176
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/30b38176
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/30b38176
Branch: refs/heads/two-dot-o
Commit: 30b38176ec960d40f9feece7c695c224f6b83528
Parents: fc79628
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Nov 24 08:55:47 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Nov 24 08:55:47 2014 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/index/IndexFig.java | 2 +-
.../usergrid/persistence/index/IndexIdentifier.java | 2 +-
.../persistence/index/impl/EsEntityIndexImpl.java | 5 ++++-
.../persistence/index/impl/EntityIndexTest.java | 12 ++++++++++++
4 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/30b38176/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index c64ddff..ea0a4f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -37,7 +37,7 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
- public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch_index_postfix";
+ public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch.alias_postfix";
public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/30b38176/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index c65a740..58e6ee5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -40,7 +40,7 @@ public class IndexIdentifier{
* @return
*/
public String getAlias() {
- return getIndexBase() + config.getAliasPostfix();
+ return getIndexBase() + "_" + config.getAliasPostfix();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/30b38176/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index aab8301..f9623ce 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -180,12 +180,15 @@ public class EsEntityIndexImpl implements EntityIndex {
if(!admin.indices().exists(new IndicesExistsRequest(indexVersionName)).actionGet().isExists()) {
final CreateIndexResponse cir = admin.indices().prepareCreate(indexVersionName).setSettings(settings).execute().actionGet();
logger.info( "Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged() );
+ }else{
+ logger.info( " Index Name [{}] already exists", indexVersionName);
}
//check if alias exists and get the alias
if(!admin.indices().aliasesExist(new GetAliasesRequest(aliasName)).actionGet().exists()) {
final Boolean isAck = admin.indices().prepareAliases().addAlias(indexVersionName, aliasName).execute().actionGet().isAcknowledged();
logger.info( "Created new Alias Name [{}] ACK=[{}]", aliasName, isAck);
-
+ }else{
+ logger.info( " Alias Name [{}] already exists", aliasName);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/30b38176/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a30e222..f8bd30f 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -136,6 +136,18 @@ public class EntityIndexTest extends BaseIT {
testQueries( indexScope, searchTypes, entityIndex );
}
+ @Test
+ public void testMultipleIndexInitializations(){
+ Id appId = new SimpleId( "application" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+ for(int i=0;i<10;i++) {
+ entityIndex.initializeIndex();
+ }
+
+ }
@Test
public void testDeindex() {
[14/22] incubator-usergrid git commit: bower fix
Posted by to...@apache.org.
bower fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/72c165e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/72c165e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/72c165e4
Branch: refs/heads/two-dot-o
Commit: 72c165e498d4d9a2dce7f440b5ff825c12f2c498
Parents: 4c12b6b
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Nov 26 10:02:50 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Nov 26 10:02:50 2014 -0700
----------------------------------------------------------------------
portal/Gruntfile.js | 193 ++++++++++++++++++++++++++++++-----------------
portal/build.sh | 2 -
2 files changed, 123 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/72c165e4/portal/Gruntfile.js
----------------------------------------------------------------------
diff --git a/portal/Gruntfile.js b/portal/Gruntfile.js
index f768b61..c00d961 100644
--- a/portal/Gruntfile.js
+++ b/portal/Gruntfile.js
@@ -18,8 +18,8 @@
*/
var bower = require('./bower.json');
-var distPath = 'dist/'+bower.name,
- coveragePath = 'dist-cov/'+bower.name,
+var distPath = 'dist/' + bower.name,
+ coveragePath = 'dist-cov/' + bower.name,
libsFile = 'js/generated/usergrid-libs.min.js',
devFile = 'js/generated/usergrid-dev.js',
mainFile = 'js/generated/usergrid.min.js',
@@ -28,7 +28,8 @@ var distPath = 'dist/'+bower.name,
coverageFile = 'test/coverage/instrument/js/usergrid-coverage.min.js',
distName = bower.name,
- licenseHeader =' /**\n \
+ licenseHeader =
+ ' /**\n \
Licensed to the Apache Software Foundation (ASF) under one\n \
or more contributor license agreements. See the NOTICE file\n \
distributed with this work for additional information \n \
@@ -46,8 +47,10 @@ var distPath = 'dist/'+bower.name,
specific language governing permissions and limitations\n \
under the License.\n \
*/\n';
-console.warn('to run e2e tests you need to have a running instance of webdriver, 1) npm install protractor -g -> 2) webdriver-manager start --standalone');
-module.exports = function (grunt) {
+console.warn(
+ 'to run e2e tests you need to have a running instance of webdriver, 1) npm install protractor -g -> 2) webdriver-manager start --standalone'
+);
+module.exports = function(grunt) {
// Project configuration.
@@ -56,16 +59,19 @@ module.exports = function (grunt) {
uglify: {
options: {
- banner: licenseHeader + '\n /*! <%= pkg.name %>@<%= pkg.version %> */\n'
+ banner: licenseHeader +
+ '\n /*! <%= pkg.name %>@<%= pkg.version %> */\n'
},
- 'usergrid-libs':{
+ 'usergrid-libs': {
options: {
mangle: false,
- compress: {warnings:false},
+ compress: {
+ warnings: false
+ },
beautify: false
},
- files:{
- 'js/generated/usergrid-libs.min.js':[
+ files: {
+ 'js/generated/usergrid-libs.min.js': [
'js/libs/jquery/jquery-1.9.1.min.js',
'js/libs/jquery/jquery-migrate-1.1.1.min.js',
'js/libs/jquery/jquery.sparkline.min.js',
@@ -110,20 +116,22 @@ module.exports = function (grunt) {
},
files: {
'test/coverage/instrument/js/usergrid-coverage.min.js': [
- coverageDir+'js/app.js',
- coverageDir+'js/**/*.js',
+ coverageDir + 'js/app.js',
+ coverageDir + 'js/**/*.js',
'js/generated-templates/templates.js',
- '!'+coverageDir+'js/config.js',
- '!'+coverageDir+'js/libs/**/*.js',
- '!'+coverageDir+'js/generated/*.js',
- '!'+coverageDir+'js/usergrid-coverage.min.js'
+ '!' + coverageDir + 'js/config.js',
+ '!' + coverageDir + 'js/libs/**/*.js',
+ '!' + coverageDir + 'js/generated/*.js',
+ '!' + coverageDir + 'js/usergrid-coverage.min.js'
]
}
},
'usergrid-coverage-min': {
options: {
mangle: false,
- compress: {warnings:false},
+ compress: {
+ warnings: false
+ },
beautify: false
},
files: {
@@ -135,7 +143,9 @@ module.exports = function (grunt) {
'usergrid': {
options: {
mangle: false,
- compress: {warnings:false},
+ compress: {
+ warnings: false
+ },
beautify: false
},
files: {
@@ -151,10 +161,12 @@ module.exports = function (grunt) {
options: {
base: 'js/'
},
- src: ['js/**/*.html','!**/index*'],
+ src: ['js/**/*.html', '!**/index*'],
dest: templateFile,
- options: {
- url: function(url) { return url.replace('js/', ''); }
+ options: {
+ url: function(url) {
+ return url.replace('js/', '');
+ }
}
}
},
@@ -188,8 +200,8 @@ module.exports = function (grunt) {
options: {
target: 'http://localhost:3000/index-debug.html', // target url to open
open: 'http://localhost:3000/index-debug.html',
- hostname:'*',
- livereload:true,
+ hostname: '*',
+ livereload: true,
port: 3000,
base: ''
}
@@ -239,13 +251,13 @@ module.exports = function (grunt) {
keepAlive: true, // If false, the grunt process stops when the test fails.
noColor: false, // If true, protractor will not use colors in its output.
args: {
- baseUrl:'http://localhost:3005/'
+ baseUrl: 'http://localhost:3005/'
}
},
phantom: {
options: {
args: {
- baseUrl:'http://localhost:3005/',
+ baseUrl: 'http://localhost:3005/',
// Arguments passed to the command
'browser': 'phantomjs'
}
@@ -254,7 +266,7 @@ module.exports = function (grunt) {
chrome: {
options: {
args: {
- baseUrl:'http://localhost:3006/?noHelp=true',
+ baseUrl: 'http://localhost:3006/?noHelp=true',
// Arguments passed to the command
'browser': 'chrome'
}
@@ -263,73 +275,103 @@ module.exports = function (grunt) {
firefox: {
options: {
args: {
- baseUrl:'http://localhost:3007/',
+ baseUrl: 'http://localhost:3007/',
// Arguments passed to the command
'browser': 'firefox'
}
}
}
},
- copy:{
- coverage:{
- files:[
- {
- src:['js/*.min.js','js/libs/**','sdk/**','js/charts/*.json','css/**','img/**','js/libs/**','config.js','bower_components/**'],
- dest:coveragePath,
- expand:true
- },
- {
- src:['js/*.min.js'],
- dest:coveragePath,
- cwd: coverageDir,
- expand:true
- }
- ]
+ copy: {
+ coverage: {
+ files: [{
+ src: ['js/*.min.js', 'js/libs/**', 'sdk/**',
+ 'js/charts/*.json', 'css/**', 'img/**', 'js/libs/**',
+ 'config.js', 'bower_components/**'
+ ],
+ dest: coveragePath,
+ expand: true
+ }, {
+ src: ['js/*.min.js'],
+ dest: coveragePath,
+ cwd: coverageDir,
+ expand: true
+ }]
},
- main:{
- files:[
+ main: {
+ files: [
// includes files within path
- {expand: true, src: ['*.html','config.js', '*.ico', 'helpJson.json'], dest: distPath, filter: 'isFile'},
- {expand: true, src: ['sdk/**','css/**','img/**','js/charts/*.json'], dest: distPath},
- {expand: true, src: ['js/generated/**','js/libs/**','css/**','img/**','bower_components/**'], dest: distPath}
+ {
+ expand: true,
+ src: ['*.html', 'config.js', '*.ico', 'helpJson.json'],
+ dest: distPath,
+ filter: 'isFile'
+ }, {
+ expand: true,
+ src: ['sdk/**', 'css/**', 'img/**', 'js/charts/*.json'],
+ dest: distPath
+ }, {
+ expand: true,
+ src: ['js/generated/**', 'js/libs/**', 'css/**', 'img/**',
+ 'bower_components/**'
+ ],
+ dest: distPath
+ }
]
}
},
clean: {
- build: ['dist/','dist-cov/','test/','js/generated/','js/*.min.js',templateFile,'index.html','index-debug.html'],
- coverage: ['reports/']
+ build: ['dist/', 'dist-cov/', 'test/', 'js/generated/',
+ 'js/*.min.js', templateFile, 'index.html', 'index-debug.html'
+ ],
+ coverage: ['reports/']
},
dom_munger: {
main: {
options: {
- update: {selector:'#main-script',attribute:'src',value:mainFile}
+ update: {
+ selector: '#main-script',
+ attribute: 'src',
+ value: mainFile
+ }
},
- src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
- dest: 'index.html' //update the dist/index.html (the src index.html is copied there)
+ src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
+ dest: 'index.html' //update the dist/index.html (the src index.html is copied there)
},
dev: {
options: {
- update: {selector:'#main-script',attribute:'src',value:devFile}
+ update: {
+ selector: '#main-script',
+ attribute: 'src',
+ value: devFile
+ }
},
- src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
- dest: 'index-debug.html' //update the dist/index.html (the src index.html is copied there)
+ src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
+ dest: 'index-debug.html' //update the dist/index.html (the src index.html is copied there)
},
coverage: {
options: {
- update: {selector:'#main-script',attribute:'src',value:'js/usergrid-coverage.min.js'}
+ update: {
+ selector: '#main-script',
+ attribute: 'src',
+ value: 'js/usergrid-coverage.min.js'
+ }
},
- src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
- dest: coveragePath+'/index.html' //update the dist/index.html (the src index.html is copied there)
+ src: 'index-template.html', //update the dist/index.html (the src index.html is copied there)
+ dest: coveragePath + '/index.html' //update the dist/index.html (the src index.html is copied there)
}
},
bower: {
install: {
- options:{
- cleanup:false,
- copy:false
+ options: {
+ cleanup: false,
+ copy: false,
+ bowerOptions: {
+ forceLatest: true
+ }
}
}
},
@@ -371,17 +413,28 @@ module.exports = function (grunt) {
grunt.registerTask('validate', ['jshint', 'complexity']);
grunt.registerTask('report', ['build', 'coverage']);
- grunt.registerTask('build', ['clean:build','bower:install','ngtemplates', 'uglify','cssmin','dom_munger','karma:unit','copy']);
- grunt.registerTask('build-dev', [ 'build']);
- grunt.registerTask('build-coverage', [ 'ngtemplates','instrument','uglify:usergrid-coverage','uglify:usergrid-coverage-min', 'cssmin','dom_munger', 'copy:coverage']);
+ grunt.registerTask('build', ['clean:build', 'bower:install', 'ngtemplates',
+ 'uglify', 'cssmin', 'dom_munger', 'karma:unit', 'copy'
+ ]);
+ grunt.registerTask('build-dev', ['build']);
+ grunt.registerTask('build-coverage', ['ngtemplates', 'instrument',
+ 'uglify:usergrid-coverage', 'uglify:usergrid-coverage-min', 'cssmin',
+ 'dom_munger', 'copy:coverage'
+ ]);
grunt.registerTask('default', ['build']);
- grunt.registerTask('e2e', ['connect:e2e-phantom','protractor:phantom']);
- grunt.registerTask('e2e-chrome', ['connect:e2e-chrome','protractor:chrome']);
- grunt.registerTask('e2e-coverage', ['clean:coverage', 'connect:e2e-coverage','protractor:coverage']);
- grunt.registerTask('e2e-coverage-chrome', ['clean:coverage', 'connect:e2e-coverage-chrome','protractor:chrome', 'makeReport']);
- grunt.registerTask('e2e-firefox', ['connect:e2e-firefox','protractor:firefox']);
- grunt.registerTask('no-monitoring', ['build','clean:perf','karma:unit']);
+ grunt.registerTask('e2e', ['connect:e2e-phantom', 'protractor:phantom']);
+ grunt.registerTask('e2e-chrome', ['connect:e2e-chrome', 'protractor:chrome']);
+ grunt.registerTask('e2e-coverage', ['clean:coverage',
+ 'connect:e2e-coverage', 'protractor:coverage'
+ ]);
+ grunt.registerTask('e2e-coverage-chrome', ['clean:coverage',
+ 'connect:e2e-coverage-chrome', 'protractor:chrome', 'makeReport'
+ ]);
+ grunt.registerTask('e2e-firefox', ['connect:e2e-firefox',
+ 'protractor:firefox'
+ ]);
+ grunt.registerTask('no-monitoring', ['build', 'clean:perf', 'karma:unit']);
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/72c165e4/portal/build.sh
----------------------------------------------------------------------
diff --git a/portal/build.sh b/portal/build.sh
index fdfb28b..e467ffd 100755
--- a/portal/build.sh
+++ b/portal/build.sh
@@ -50,5 +50,3 @@ echo or run it now using the following command:
echo grunt dev
echo ###
echo Happy Usergriding!
-
-
[02/22] incubator-usergrid git commit: add alias support
Posted by to...@apache.org.
add alias support
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/55110a5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/55110a5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/55110a5d
Branch: refs/heads/two-dot-o
Commit: 55110a5d87f33d7e7a1bc7c0752b3a7b4baacf17
Parents: 7b08ea5
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 13:57:11 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 13:57:11 2014 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 4 +-
.../persistence/index/impl/IndexingUtils.java | 65 +++++++++++---------
3 files changed, 38 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/55110a5d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 9f50ccd..3fb3340 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -98,7 +98,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
- this.aliasName = IndexingUtils.createAliasName(createIndexName( config.getIndexPrefix(), applicationScope ),config);
+ this.aliasName = IndexingUtils.createAliasName( config, applicationScope );
this.refresh = config.isForcedRefresh();
this.autoFlushSize = autoFlushSize;
initBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/55110a5d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 7407e49..8555b1c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -137,7 +137,7 @@ public class EsEntityIndexImpl implements EntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.aliasName = IndexingUtils.createAliasName(IndexingUtils.createIndexName(config.getIndexPrefix(), appScope),config);
+ this.aliasName = IndexingUtils.createAliasName(config,appScope);
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
@@ -178,7 +178,7 @@ public class EsEntityIndexImpl implements EntityIndex {
private void createIndexAndAlias(AdminClient admin, Settings settings) {
String indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), applicationScope);
- String indexVersionName = IndexingUtils.getIndexName(indexName, 0);
+ String indexVersionName = IndexingUtils.createIndexName(indexName, 0);
final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
//check if alias exists and get the alias
admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/55110a5d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index dcd496b..3aa4f39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -98,15 +98,25 @@ public class IndexingUtils {
* @param applicationScope
* @return
*/
- public static String createIndexName(
- String prefix, ApplicationScope applicationScope) {
+ public static String createIndexBaseName(String prefix, ApplicationScope applicationScope) {
StringBuilder sb = new StringBuilder();
sb.append( prefix ).append( SEPARATOR );
idString( sb, applicationScope.getApplication() );
return sb.toString();
}
+ public static String createIndexName(String indexName, int i) {
+ if(i>0) {
+ return indexName + "_v" + (i + 1);
+ }else{
+ return indexName;
+ }
+ }
+ public static String createAliasName( IndexFig config,ApplicationScope applicationScope) {
+ String indexName =IndexingUtils.createIndexBaseName(config.getIndexPrefix(), applicationScope);
+ return indexName + config.getAliasPostfix();
+ }
/**
* Create the index doc from the given entity
@@ -155,54 +165,54 @@ public class IndexingUtils {
.startObject( type )
- .startArray( "dynamic_templates" )
+ .startArray("dynamic_templates")
//we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" )
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" ).endObject()
+ .startObject("context_template")
+ .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME)
+ .field("match_mapping_type", "string")
+ .startObject("mapping").field( "type", "string" )
+ .field("index", "not_analyzed").endObject()
.endObject()
.endObject()
// any string with field name that starts with sa_ gets analyzed
.startObject()
- .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" )
- .field( "index", "analyzed" ).endObject().endObject()
+ .startObject("template_1").field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field("match_mapping_type", "string").startObject( "mapping" ).field( "type", "string" )
+ .field("index", "analyzed").endObject().endObject()
.endObject()
// all other strings are not analyzed
.startObject()
- .startObject( "template_2" )
+ .startObject("template_2")
//todo, should be string prefix, remove 2 field mapping
- .field( "match", "*" ).field( "match_mapping_type", "string" ).startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" )
+ .field("match", "*").field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field("type", "string")
+ .field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
// fields names starting with go_ get geo-indexed
.startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
+ .startObject("template_3")
+ .field("match", GEO_PREFIX + "location")
+ .startObject("mapping")
+ .field("type", "geo_point")
.endObject()
.endObject()
.endObject()
@@ -218,11 +228,6 @@ public class IndexingUtils {
return builder;
}
- public static String createAliasName(String indexName, IndexFig fig) {
- return indexName + fig.getAliasPostfix();
- }
- public static String getIndexName(String indexName, int i) {
- return indexName+"_v"+(i+1);
- }
+
}
[18/22] incubator-usergrid git commit: ignore test for now
Posted by to...@apache.org.
ignore test for now
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5c2235fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5c2235fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5c2235fc
Branch: refs/heads/two-dot-o
Commit: 5c2235fcfe4f2b8cdbebadd3d36f84c5f534bf52
Parents: d28af0d
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 2 15:21:38 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 2 15:21:38 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/IndexResourceIT.java | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c2235fc/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
index bc33baa..9bf0813 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
@@ -21,6 +21,7 @@
package org.apache.usergrid.rest;
import com.fasterxml.jackson.databind.JsonNode;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -32,9 +33,10 @@ import java.util.Map;
import java.util.UUID;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
/**
- * Classy class class.
+ * test index creation
*/
public class IndexResourceIT extends AbstractRestIT {
@@ -48,6 +50,7 @@ public class IndexResourceIT extends AbstractRestIT {
}
+ @Ignore( "will finish when tests are working from rest" )
@Test
public void TestAddIndex() throws Exception{
String superToken = superAdminToken();
@@ -59,10 +62,16 @@ public class IndexResourceIT extends AbstractRestIT {
UUID appId = this.context.getAppUuid();
// change the password as admin. The old password isn't required
- JsonNode node = mapper.readTree( resource().path( "/index/"+appId ).queryParam( "access_token", superToken )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- .post( String.class, data ));
-
+ JsonNode node = null;
+ try {
+ node = mapper.readTree(resource().path("/system/index/" + appId)
+ .queryParam("access_token", superToken)
+ .accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON_TYPE)
+ .post(String.class, data));
+ }catch (Exception e){
+ LOG.error("failed", e);
+ fail(e.toString());
+ }
assertNull( getError( node ) );
refreshIndex("test-organization", "test-app");
[08/22] incubator-usergrid git commit: Merge branch 'two-dot-o' into
index-alias
Posted by to...@apache.org.
Merge branch 'two-dot-o' into index-alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/486c0d25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/486c0d25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/486c0d25
Branch: refs/heads/two-dot-o
Commit: 486c0d2512153c0ffea57674216b63eb228d2317
Parents: 30b3817 2586e9f
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Nov 24 09:10:23 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Nov 24 09:10:23 2014 -0700
----------------------------------------------------------------------
.../corepersistence/util/CpEntityMapUtils.java | 62 ++++---
.../org/apache/usergrid/CoreApplication.java | 17 +-
.../migration/EntityDataMigrationIT.java | 15 +-
.../migration/EntityTypeMappingMigrationIT.java | 17 ++
.../migration/GraphShardVersionMigrationIT.java | 18 +-
.../migration/MigrationTestRule.java | 101 ++++++++++
.../persistence/model/field/LocationField.java | 3 -
.../index/impl/EntityIndexMapUtils.java | 4 +-
.../collection/groups/GroupResourceIT.java | 182 +-----------------
.../applications/queries/GeoPagingTest.java | 41 +++--
.../applications/queries/basicGeoTests.java | 184 +++++++++++++++++++
11 files changed, 406 insertions(+), 238 deletions(-)
----------------------------------------------------------------------
[16/22] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into index-alias
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into index-alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e2c1a539
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e2c1a539
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e2c1a539
Branch: refs/heads/two-dot-o
Commit: e2c1a5390a286a57049c3e308b0684406db54b20
Parents: 360a78e de455c3
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Nov 26 21:28:27 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Nov 26 21:28:27 2014 -0700
----------------------------------------------------------------------
.../rest/management/users/UsersResource.java | 38 ++-
.../applications/queries/BasicGeoTests.java | 268 +++++++++++++++++++
.../applications/queries/basicGeoTests.java | 184 -------------
3 files changed, 292 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
[10/22] incubator-usergrid git commit: add index to alias methods
Posted by to...@apache.org.
add index to alias methods
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b376cfd1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b376cfd1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b376cfd1
Branch: refs/heads/two-dot-o
Commit: b376cfd13ef76d8f2011201c65a14585d2055023
Parents: af16be2
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Nov 25 13:49:24 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Nov 25 13:49:24 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/EntityIndex.java | 12 +++
.../persistence/index/IndexIdentifier.java | 8 +-
.../index/impl/EsEntityIndexImpl.java | 95 +++++++++++---------
.../persistence/index/impl/EntityIndexTest.java | 83 +++++++++++------
4 files changed, 127 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b376cfd1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 88498b3..30dc3fc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -37,11 +37,23 @@ public interface EntityIndex {
public void initializeIndex();
/**
+ * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
+ * @param indexSuffix index name
+ */
+ public void createIndexAddToAlias(final String indexSuffix, final int numShards, final int numReplicas);
+
+ /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
/**
+ * Add alias to index, will remove old index from write alias
+ * @param indexSuffix must be different than current index
+ */
+ public void addAlias(final String indexSuffix);
+
+ /**
* Execute query in Usergrid syntax.
*/
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b376cfd1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 82a6023..54a02d6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -45,12 +45,12 @@ public class IndexIdentifier{
/**
* Get index name, send in additional parameter to add incremental indexes
- * @param version
+ * @param suffix
* @return
*/
- public String getIndex(int version) {
- if (version > 0) {
- return getIndexBase() + "_v" + (version + 1);
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return getIndexBase() + "_" + (suffix + 1);
} else {
return getIndexBase();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b376cfd1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 540aec4..e7cbd2e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -36,6 +37,8 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -115,11 +118,8 @@ public class EsEntityIndexImpl implements EntityIndex {
@Inject
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
- final EsProvider provider ) {
-
+ public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider ) {
ValidationUtils.validateApplicationScope( appScope );
-
this.applicationScope = appScope;
this.esProvider = provider;
this.config = config;
@@ -129,58 +129,67 @@ public class EsEntityIndexImpl implements EntityIndex {
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
-
@Override
public void initializeIndex() {
+ final int numberOfShards = config.getNumberOfShards();
+ final int numberOfReplicas = config.getNumberOfReplicas();
+ createIndexAddToAlias(null,numberOfShards,numberOfReplicas);
+ }
+ @Override
+ public void createIndexAddToAlias(final String indexSuffix, final int numberOfShards, final int numberOfReplicas) {
try {
- if ( !mappingsCreated.getAndSet( true ) ) {
+
+ if (!mappingsCreated.getAndSet(true)) {
createMappings();
}
- createIndexAndAlias();
+ //get index name with suffix attached
+ String indexName = indexIdentifier.getIndex(indexSuffix);
- // create the document, this ensures the index is ready
+ //Create index
+ try {
+ final AdminClient admin = esProvider.getClient().admin();
+ Settings settings = ImmutableSettings.settingsBuilder().put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", numberOfReplicas).build();
+ final CreateIndexResponse cir = admin.indices().prepareCreate(indexName).setSettings(settings).execute().actionGet();
+ logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
+ } catch (IndexAlreadyExistsException e) {
+ logger.info("Index Name [{}] already exists", indexName);
+ }
- // Immediately create a document and remove it to ensure the entire cluster is ready
- // to receive documents. Occasionally we see errors.
- // See this post: http://s.apache.org/index-missing-exception
+ addAlias(indexSuffix);
testNewIndex();
- }
- catch ( IndexAlreadyExistsException expected ) {
+ } catch (IndexAlreadyExistsException expected) {
// this is expected to happen if index already exists, it's a no-op and swallow
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to initialize index", e );
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to initialize index", e);
}
}
- /**
- * Create the index and alias
- */
- private void createIndexAndAlias() {
- final int numberOfShards = config.getNumberOfShards();
- final int numberOfReplicas = config.getNumberOfReplicas();
- final AdminClient admin = esProvider.getClient().admin();
-
- Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
- .put( "index.number_of_replicas", numberOfReplicas ).build();
- //TODO:swallow exception, look into setting up routing rules
- String indexVersionName = indexIdentifier.getIndex(0);
+ @Override
+ public void addAlias(final String indexSuffix) {
try {
- final CreateIndexResponse cir = admin.indices().prepareCreate(indexVersionName).setSettings(settings).execute().actionGet();
- logger.info("Created new Index Name [{}] ACK=[{}]", indexVersionName, cir.isAcknowledged());
- }catch(IndexAlreadyExistsException e){
- logger.info("Index Name [{}] already exists",indexVersionName);
- }
- try { //check if alias exists and get the alias
- Boolean isAck = admin.indices().prepareAliases().addAlias(indexVersionName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
- logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+ Boolean isAck;
+ String indexName = indexIdentifier.getIndex(indexSuffix);
+ final AdminClient adminClient = esProvider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
+ String[] indexNames = aliasMap.keys().toArray(String.class);
+ for(String currentIndex : indexNames){
+ isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
- isAck = admin.indices().prepareAliases().addAlias(indexVersionName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ }
+ //add read alias
+ isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+ //add write alias
+ isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
+
} catch (Exception e) {
logger.warn("Failed to create alias ", e);
}
@@ -193,6 +202,10 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
private void testNewIndex() {
+ // create the document, this ensures the index is ready
+ // Immediately create a document and remove it to ensure the entire cluster is ready
+ // to receive documents. Occasionally we see errors.
+ // See this post: http://s.apache.org/index-missing-exception
logger.info( "Refreshing Created new Index Name [{}]", alias);
@@ -430,7 +443,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public int getPendingTasks() {
final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin()
- .cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
+ .cluster().pendingClusterTasks(new PendingClusterTasksRequest()).actionGet();
return tasksResponse.pendingTasks().size();
}
@@ -475,7 +488,7 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexIdentifier.getIndex(0) ).get();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexIdentifier.getIndex(null) ).get();
if ( response.isAcknowledged() ) {
logger.info( "Deleted index: " + alias);
}
@@ -519,7 +532,7 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse chr = esProvider.getClient().admin()
- .cluster().health( new ClusterHealthRequest() ).get();
+ .cluster().health(new ClusterHealthRequest()).get();
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
@@ -539,7 +552,7 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(0)}))
+ .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)}))
.get();
return Health.valueOf( chr.getStatus().name() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b376cfd1/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index f8bd30f..8ea8835 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -76,25 +76,67 @@ public class EntityIndexTest extends BaseIT {
@Test
public void testIndex() throws IOException {
-
- final int MAX_ENTITIES = 100;
-
Id appId = new SimpleId( "application" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+ entityIndex.initializeIndex();
final String entityType = "thing";
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,0);
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ entityIndex.refresh();
+ testQueries( indexScope, searchTypes, entityIndex );
+ }
+
+ @Test
+ public void testMultipleIndexInitializations(){
+ Id appId = new SimpleId( "application" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
entityIndex.initializeIndex();
+ for(int i=0;i<10;i++) {
+ entityIndex.initializeIndex();
+ }
+
+ }
+
+ @Test
+ public void testAddMultipleIndexes() throws IOException {
+ Id appId = new SimpleId( "application" );
- InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+ entityIndex.initializeIndex();
+
+ final String entityType = "thing";
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,0);
+
+ entityIndex.refresh();
+
+ testQueries( indexScope, searchTypes, entityIndex );
+
+ entityIndex.createIndexAddToAlias("v2",1,0);
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
+
+ testQueries( indexScope, searchTypes, entityIndex );
+
+ }
+
+ private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope, String filePath,final int max,final int startIndex) throws IOException {
+ InputStream is = this.getClass().getResourceAsStream( filePath );
ObjectMapper mapper = new ObjectMapper();
List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
@@ -104,13 +146,19 @@ public class EntityIndexTest extends BaseIT {
final EntityIndexBatch batch = entityIndex.createBatch();
+ if(startIndex > 0){
+ for(int i =0; i<startIndex;i++){
+ sampleJson.remove(0);
+ }
+ }
+
for ( Object o : sampleJson ) {
Map<String, Object> item = ( Map<String, Object> ) o;
Entity entity = new Entity( entityType );
- entity = EntityIndexMapUtils.fromMap( entity, item );
- EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+ entity = EntityIndexMapUtils.fromMap(entity, item);
+ EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
batch.index( indexScope, entity );
@@ -120,7 +168,7 @@ public class EntityIndexTest extends BaseIT {
- if ( count++ > MAX_ENTITIES ) {
+ if ( count++ > max ) {
break;
}
}
@@ -129,25 +177,8 @@ public class EntityIndexTest extends BaseIT {
timer.stop();
log.info( "Total time to index {} entries {}ms, average {}ms/entry",
new Object[] { count, timer.getTime(), timer.getTime() / count } );
-
- entityIndex.refresh();
-
-
- testQueries( indexScope, searchTypes, entityIndex );
}
- @Test
- public void testMultipleIndexInitializations(){
- Id appId = new SimpleId( "application" );
-
- ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
- EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
- for(int i=0;i<10;i++) {
- entityIndex.initializeIndex();
- }
-
- }
@Test
public void testDeindex() {
[19/22] incubator-usergrid git commit: fix index name
Posted by to...@apache.org.
fix index name
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c083e6b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c083e6b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c083e6b4
Branch: refs/heads/two-dot-o
Commit: c083e6b424ed35a2e60748843cf6d83430c40ccd
Parents: 5c2235f
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 2 16:56:58 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 2 16:56:58 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 4 ++--
.../HybridEntityManagerFactory.java | 4 ++--
.../usergrid/persistence/EntityManagerFactory.java | 5 +++--
.../cassandra/EntityManagerFactoryImpl.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 5 +++--
.../persistence/index/IndexIdentifier.java | 2 +-
.../persistence/index/impl/EsEntityIndexImpl.java | 17 ++++-------------
.../persistence/index/impl/EntityIndexTest.java | 6 +-----
.../org/apache/usergrid/rest/IndexResource.java | 13 +++++++++----
.../org/apache/usergrid/rest/AbstractRestIT.java | 2 +-
10 files changed, 27 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5a09423..f6b0df8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -701,8 +701,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
- public void addIndex(final UUID applicationId,final String indexSuffix,final Map<String,Object> config){
- getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope( applicationId )).addIndex(indexSuffix,config);
+ public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas){
+ getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope( applicationId )).addIndex(indexSuffix,shards,replicas);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 0f1fbed..42dc805 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -198,8 +198,8 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
}
@Override
- public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
- factory.addIndex(appId,suffix,config);
+ public void addIndex(UUID appId, String suffix,final int shards,final int replicas) {
+ factory.addIndex(appId,suffix,shards,replicas);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 33413a0..3d4827b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -165,9 +165,10 @@ public interface EntityManagerFactory {
* Add a new index to the application for scale
* @param appId application id
* @param suffix unique indentifier for additional index
- * @param config {"replicas":num,"shards":num}
+ * @param shards number of shards
+ * @param replicas number of replicas
*/
- public void addIndex(final UUID appId,final String suffix, Map<String,Object> config);
+ public void addIndex(final UUID appId,final String suffix,final int shards,final int replicas);
public Health getEntityStoreHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 68abf2e..f073da7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -472,7 +472,7 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
- public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
+ public void addIndex(UUID appId, String suffix,final int shards,final int replicas) {
throw new UnsupportedOperationException("Not supported in v1");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 93dd3ea..4a653b3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -41,9 +41,10 @@ public interface EntityIndex {
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
- * @param config {"replicas":num,"shards":num}
+ * @param shards
+ * @param replicas
*/
- public void addIndex(final String indexSuffix, final Map<String,Object> config);
+ public void addIndex(final String indexSuffix, final int shards, final int replicas);
/**
* Create the index batch.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 54a02d6..86b5611 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -50,7 +50,7 @@ public class IndexIdentifier{
*/
public String getIndex(String suffix) {
if (suffix != null) {
- return getIndexBase() + "_" + (suffix + 1);
+ return getIndexBase() + "_" + suffix;
} else {
return getIndexBase();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 0b1cb0c..d546092 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -139,17 +139,8 @@ public class EsEntityIndexImpl implements EntityIndex {
}
@Override
- public void addIndex(final String indexSuffix, final Map<String,Object> config) {
- if(!config.containsKey("replicas") || !config.containsKey("shards") ||
- !(config.get("replicas") instanceof Integer) || !(config.get("shards") instanceof Integer)){
- throw new IllegalArgumentException("config must contains 'replicas' of type int and 'shards' of type int");
- }
- String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
- addIndex(normalizedSuffix,(int) config.get("shards"),(int)config.get("replicas"));
- }
-
- private void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
-
+ public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
+ String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
try {
if (!mappingsCreated.getAndSet(true)) {
@@ -157,7 +148,7 @@ public class EsEntityIndexImpl implements EntityIndex {
}
//get index name with suffix attached
- String indexName = indexIdentifier.getIndex(indexSuffix);
+ String indexName = indexIdentifier.getIndex(normalizedSuffix);
//Create index
try {
@@ -170,7 +161,7 @@ public class EsEntityIndexImpl implements EntityIndex {
logger.info("Index Name [{}] already exists", indexName);
}
- addAlias(indexSuffix);
+ addAlias(normalizedSuffix);
testNewIndex();
} catch (IndexAlreadyExistsException expected) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 360347a..13d1552 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -127,11 +127,7 @@ public class EntityIndexTest extends BaseIT {
testQueries( indexScope, searchTypes, entityIndex );
- HashMap<String,Object> config = new HashMap<>();
- config.put("replicas",0);
- config.put("shards",1);
-
- entityIndex.addIndex("v2", config);
+ entityIndex.addIndex("v2", 1,0);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index 02e2923..5cbc499 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -188,13 +188,18 @@ public class IndexResource extends AbstractContextResource {
ApiResponse response = createApiResponse();
final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
- if(config.containsKey("indexSuffix")) {
- emf.addIndex(appId, config.get("indexSuffix").toString(), config);
- response.setAction("Add index to alias");
+ if(!config.containsKey("replicas") || !config.containsKey("shards") ||
+ !(config.get("replicas") instanceof Integer) || !(config.get("shards") instanceof Integer)){
+ throw new IllegalArgumentException("body must contains 'replicas' of type int and 'shards' of type int");
}
- else{
+
+ if(!config.containsKey("indexSuffix")) {
throw new IllegalArgumentException("Please add an indexSuffix to your post");
}
+
+ emf.addIndex(appId, config.get("indexSuffix").toString(), (int) config.get("shards"),(int) config.get("replicas"));
+ response.setAction("Add index to alias");
+
return new JSONWithPadding(response, callback);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c083e6b4/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
index 1a18568..6ccb923 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
@@ -276,7 +276,7 @@ public abstract class AbstractRestIT extends JerseyTest {
/** Get the super user's access token */
protected String superAdminToken() {
- return mgmtToken( "superuser", "superpassword" );
+ return mgmtToken( "superuser", "test" );
}
[20/22] incubator-usergrid git commit: fix write alias issue
Posted by to...@apache.org.
fix write alias issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/887bb476
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/887bb476
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/887bb476
Branch: refs/heads/two-dot-o
Commit: 887bb476e3616d8a972c761fcb1e0051d530cbef
Parents: c083e6b
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 2 17:59:33 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 2 17:59:33 2014 -0700
----------------------------------------------------------------------
portal/config.js | 94 ++++++++++++++------
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
2 files changed, 70 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/887bb476/portal/config.js
----------------------------------------------------------------------
diff --git a/portal/config.js b/portal/config.js
index 70138f4..36d68ed 100644
--- a/portal/config.js
+++ b/portal/config.js
@@ -1,4 +1,3 @@
-
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -28,42 +27,87 @@ Usergrid.overrideUrl = 'http://localhost:8080/';
//Usergrid.overrideUrl = 'https://api.usergrid.com/';
Usergrid.options = {
- client:{
- requiresDeveloperKey:false
- // apiKey:'123456'
+ client: {
+ requiresDeveloperKey: false
+ // apiKey:'123456'
},
- showAutoRefresh:true,
- autoUpdateTimer:61, //seconds
- menuItems:[
- {path:'#!/org-overview', active:true,pic:'🕪',title:'Org Administration'},
- {path:'#!/app-overview/summary',pic:'',title:'App Overview'},
- {path:'#!/users',pic:'👤',title:'Users'},
- {path:'#!/groups',pic:'👥',title:'Groups'},
- {path:'#!/roles',pic:'',title:'Roles'},
- {path:'#!/data',pic:'📸',title:'Data'},
- {path:'#!/activities',pic:'',title:'Activities'},
- {path:'#!/push/getStarted',pic:'',title:'Push', items:[
- {path:'#!/push/getStarted',pic:'',title:'Get Started'},
- {path:'#!/push/configuration',pic:'⚒',title:'Configure'},
- {path:'#!/push/history',pic:'✇',title:'History'},
- {path:'#!/push/sendNotification',pic:'',title:'Send'}
- ]},
-
+ showAutoRefresh: true,
+ autoUpdateTimer: 61, //seconds
+ menuItems: [{
+ path: '#!/org-overview',
+ active: true,
+ pic: '🕪',
+ title: 'Org Administration'
+ }, {
+ path: '#!/app-overview/summary',
+ pic: '',
+ title: 'App Overview'
+ }, {
+ path: '#!/users',
+ pic: '👤',
+ title: 'Users'
+ }, {
+ path: '#!/groups',
+ pic: '👥',
+ title: 'Groups'
+ }, {
+ path: '#!/roles',
+ pic: '',
+ title: 'Roles'
+ }, {
+ path: '#!/data',
+ pic: '📸',
+ title: 'Data'
+ }, {
+ path: '#!/activities',
+ pic: '',
+ title: 'Activities'
+ }, {
+ path: '#!/push/getStarted',
+ pic: '',
+ title: 'Push',
+ items: [{
+ path: '#!/push/getStarted',
+ pic: '',
+ title: 'Get Started'
+ }, {
+ path: '#!/push/configuration',
+ pic: '⚒',
+ title: 'Configure'
+ }, {
+ path: '#!/push/history',
+ pic: '✇',
+ title: 'History'
+ }, {
+ path: '#!/push/sendNotification',
+ pic: '',
+ title: 'Send'
+ }]
+ },
+
- {path:'#!/shell',pic:'⌨',title:'Shell'}
+ {
+ path: '#!/shell',
+ pic: '⌨',
+ title: 'Shell'
+ }
]
};
Usergrid.regex = {
appNameRegex: new RegExp("^[0-9a-zA-Z.-]{3,25}$"),
usernameRegex: new RegExp("^[0-9a-zA-Z@\.\_-]{4,25}$"),
- nameRegex: new RegExp("^([0-9a-zA-Z@#$%^&!?;:.,'\"~*-:+_\[\\](){}/\\ |]{3,60})+$"),
+ nameRegex: new RegExp(
+ "^([0-9a-zA-Z@#$%^&!?;:.,'\"~*-:+_\[\\](){}/\\ |]{3,60})+$"),
roleNameRegex: new RegExp("^([0-9a-zA-Z./-]{3,25})+$"),
- emailRegex: new RegExp("^(([0-9a-zA-Z]+[_\+.-]?)+@[0-9a-zA-Z]+[0-9,a-z,A-Z,.,-]*(.){1}[a-zA-Z]{2,4})+$"),
+ emailRegex: new RegExp(
+ "^(([0-9a-zA-Z]+[_\+.-]?)+@[0-9a-zA-Z]+[0-9,a-z,A-Z,.,-]*(.){1}[a-zA-Z]{2,4})+$"
+ ),
passwordRegex: /(?=^.{8,}$)((?=.*\d)|(?=.*\W+))(?![.\n])(?=.*[A-Z])(?=.*[a-z]).*$/,
pathRegex: new RegExp("^/[a-zA-Z0-9\.\*_~-]+(\/[a-zA-Z0-9\.\*_~-]+)*$"),
titleRegex: new RegExp("[a-zA-Z0-9.!-?]+[\/]?"),
- urlRegex: new RegExp("^(http?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?$"),
+ urlRegex: new RegExp(
+ "^(http?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?$"),
zipRegex: new RegExp("^[0-9]{5}(?:-[0-9]{4})?$"),
countryRegex: new RegExp("^[A-Za-z ]{3,100}$"),
stateRegex: new RegExp("^[A-Za-z ]{2,100}$"),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/887bb476/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 0c2dec0..a3ca8a5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -182,7 +182,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- bulkRequest.add( client.prepareDelete(alias.getReadAlias(), entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
[13/22] incubator-usergrid git commit: rest path changes,
better parsing logic
Posted by to...@apache.org.
rest path changes, better parsing logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c12b6b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c12b6b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c12b6b1
Branch: refs/heads/two-dot-o
Commit: 4c12b6b1493bcd4ff92fc527db6ad0d41292f257
Parents: cfa73b4
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Nov 26 09:19:54 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Nov 26 09:19:54 2014 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 4 +++-
.../org/apache/usergrid/rest/IndexResource.java | 21 +++++++++++++-------
2 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c12b6b1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 623788e..588a5a1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.utils.StringUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@ -142,7 +143,8 @@ public class EsEntityIndexImpl implements EntityIndex {
if(!config.containsKey("replicas") || !config.containsKey("shards")){
throw new IllegalArgumentException("config must contains 'replicas' and 'shards'");
}
- addIndex(indexSuffix, (int) config.get("shards"),(int)config.get("replicas"));
+ String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
+ addIndex(normalizedSuffix, (int) config.get("shards"),(int)config.get("replicas"));
}
private void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c12b6b1/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index f95d439..02e2923 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -178,18 +178,25 @@ public class IndexResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+ @RequireSystemAccess
+ @POST
+ @Path( RootResource.APPLICATION_ID_PATH )
public JSONWithPadding addIndex(@Context UriInfo ui,
@PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "indexSuffix" ) final String indexSuffix,
- @PathParam("shards") final int shards,
- @PathParam("replicas") final int replicas,
Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) {
- final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
- emf.addIndex(appId, indexSuffix, config);
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) throws Exception{
ApiResponse response = createApiResponse();
- response.setAction("Add index to alias");
+ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
+
+ if(config.containsKey("indexSuffix")) {
+ emf.addIndex(appId, config.get("indexSuffix").toString(), config);
+ response.setAction("Add index to alias");
+ }
+ else{
+ throw new IllegalArgumentException("Please add an indexSuffix to your post");
+ }
return new JSONWithPadding(response, callback);
+
}
private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {