You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2014/12/04 21:10:37 UTC
[02/50] incubator-usergrid git commit: add alias support
add alias support
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7b08ea59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7b08ea59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7b08ea59
Branch: refs/heads/USERGRID-256-ApiResponseImplementation
Commit: 7b08ea599062ad550dfb0d9b4d05fdc0d2e8007a
Parents: 00eb339
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Nov 21 13:43:36 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Nov 21 13:43:36 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 6 ++
.../index/impl/EsEntityIndexBatchImpl.java | 8 +--
.../index/impl/EsEntityIndexImpl.java | 64 ++++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 8 +++
4 files changed, 57 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 0f405f3..c64ddff 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -37,6 +37,8 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_INDEX_PREFIX = "elasticsearch.index_prefix";
+ public static final String ELASTICSEARCH_ALIAS_POSTFIX = "elasticsearch_index_postfix";
+
public static final String ELASTICSEARCH_STARTUP = "elasticsearch.startup";
public static final String ELASTICSEARCH_NUMBER_OF_SHARDS = "elasticsearch.number_shards";
@@ -70,6 +72,10 @@ public interface IndexFig extends GuicyFig {
@Default( "usergrid" ) // no underbars allowed
@Key( ELASTICSEARCH_INDEX_PREFIX )
String getIndexPrefix();
+
+ @Default( "alias" ) // no underbars allowed
+ @Key( ELASTICSEARCH_ALIAS_POSTFIX )
+ String getAliasPostfix();
@Default( "1" ) // TODO: does this timeout get extended on each query?
@Key( QUERY_CURSOR_TIMEOUT_MINUTES )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index c4cdeeb..9f50ccd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -81,7 +81,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final boolean refresh;
- private final String indexName;
+ private final String aliasName;
private BulkRequestBuilder bulkRequest;
@@ -98,7 +98,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
- this.indexName = createIndexName( config.getIndexPrefix(), applicationScope );
+ this.aliasName = IndexingUtils.createAliasName(createIndexName( config.getIndexPrefix(), applicationScope ),config);
this.refresh = config.isForcedRefresh();
this.autoFlushSize = autoFlushSize;
initBatch();
@@ -137,7 +137,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource( entityAsMap ) );
+ bulkRequest.add( client.prepareIndex( aliasName, entityType, indexId ).setSource( entityAsMap ) );
maybeFlush();
@@ -169,7 +169,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete( aliasName, entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b1e5374..7407e49 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -28,13 +29,21 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.hppc.ObjectContainer;
+import org.elasticsearch.common.hppc.cursors.ObjectCursor;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -89,13 +98,13 @@ public class EsEntityIndexImpl implements EntityIndex {
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
+ private final String aliasName;
+
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
*/
private FailureMonitor failureMonitor;
- private final String indexName;
-
private final ApplicationScope applicationScope;
private final EsProvider esProvider;
@@ -128,7 +137,7 @@ public class EsEntityIndexImpl implements EntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexName = IndexingUtils.createIndexName( config.getIndexPrefix(), appScope );
+ this.aliasName = IndexingUtils.createAliasName(IndexingUtils.createIndexName(config.getIndexPrefix(), appScope),config);
this.failureMonitor = new FailureMonitorImpl( config, provider );
}
@@ -146,14 +155,10 @@ public class EsEntityIndexImpl implements EntityIndex {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards )
+ Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards)
.put( "index.number_of_replicas", numberOfReplicas ).build();
-
- final CreateIndexResponse cir =
- admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
-
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ createIndexAndAlias(admin, settings);
// create the document, this ensures the index is ready
@@ -171,6 +176,15 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
+ private void createIndexAndAlias(AdminClient admin, Settings settings) {
+ String indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), applicationScope);
+ String indexVersionName = IndexingUtils.getIndexName(indexName, 0);
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexVersionName ).setSettings( settings ).execute().actionGet();
+ //check if alias exists and get the alias
+ admin.indices().prepareAliases().addAlias(indexVersionName,aliasName).execute().actionGet();
+ logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@ -179,24 +193,24 @@ public class EsEntityIndexImpl implements EntityIndex {
private void testNewIndex() {
- logger.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", aliasName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+ esProvider.getClient().prepareIndex( aliasName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
.get();
logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- indexName, VERIFY_TYPE );
+ aliasName, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
+ esProvider.getClient().prepareDeleteByQuery( aliasName ).setTypes( VERIFY_TYPE )
.setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}", aliasName, VERIFY_TYPE );
return true;
}
@@ -247,7 +261,7 @@ public class EsEntityIndexImpl implements EntityIndex {
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes )
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( aliasName ).setTypes( entityTypes )
.setScroll( cursorTimeout + "m" ).setQuery( qb );
@@ -303,7 +317,7 @@ public class EsEntityIndexImpl implements EntityIndex {
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
- this.indexName, context, entityTypes, srb
+ this.aliasName, context, entityTypes, srb
} );
}
@@ -386,14 +400,14 @@ public class EsEntityIndexImpl implements EntityIndex {
public void refresh() {
- logger.info( "Refreshing Created new Index Name [{}]", indexName );
+ logger.info( "Refreshing Created new Index Name [{}]", aliasName );
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName ).execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
+ esProvider.getClient().admin().indices().prepareRefresh( aliasName ).execute().actionGet();
+ logger.debug( "Refreshed index: " + aliasName );
return true;
}
catch ( IndexMissingException e ) {
@@ -405,7 +419,7 @@ public class EsEntityIndexImpl implements EntityIndex {
doInRetry( retryOperation );
- logger.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + aliasName );
}
@@ -430,7 +444,7 @@ public class EsEntityIndexImpl implements EntityIndex {
final SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( indexName ).setTypes( searchTypes.getTypeNames() )
+ esProvider.getClient().prepareSearch( aliasName ).setTypes( searchTypes.getTypeNames() )
.setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
@@ -456,12 +470,12 @@ public class EsEntityIndexImpl implements EntityIndex {
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( aliasName ).get();
if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + aliasName );
}
else {
- logger.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + aliasName );
}
}
@@ -520,7 +534,7 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health( new ClusterHealthRequest( new String[] { indexName } ) )
+ .health( new ClusterHealthRequest( new String[] { aliasName } ) )
.get();
return Health.valueOf( chr.getStatus().name() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7b08ea59/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c7592bc..dcd496b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -217,4 +218,11 @@ public class IndexingUtils {
return builder;
}
+ public static String createAliasName(String indexName, IndexFig fig) {
+ return indexName + fig.getAliasPostfix();
+ }
+
+ public static String getIndexName(String indexName, int i) {
+ return indexName+"_v"+(i+1);
+ }
}