You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/12/16 16:00:09 UTC
[39/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ed0ea46d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ed0ea46d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ed0ea46d
Branch: refs/heads/no-source-in-es
Commit: ed0ea46dacdb49dfce7219ea544c6bfd2cd1cdfa
Parents: 532a02c 7645a79
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Dec 4 17:23:31 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Dec 4 17:23:31 2014 -0500
----------------------------------------------------------------------
portal/Gruntfile.js | 193 ++++++++-----
portal/build.sh | 2 -
portal/config.js | 94 +++++--
.../src/main/groovy/NodeRegistry.groovy | 12 +-
.../corepersistence/CpEntityManagerFactory.java | 5 +
.../HybridEntityManagerFactory.java | 5 +
.../events/EntityVersionDeletedHandler.java | 10 +-
.../persistence/EntityManagerFactory.java | 10 +
.../cassandra/EntityManagerFactoryImpl.java | 5 +
.../collection/event/EntityVersionDeleted.java | 1 -
.../usergrid/persistence/index/EntityIndex.java | 16 ++
.../usergrid/persistence/index/IndexFig.java | 6 +
.../persistence/index/IndexIdentifier.java | 88 ++++++
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexImpl.java | 280 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 20 +-
.../persistence/index/impl/EntityIndexTest.java | 75 ++++-
.../simulations/PostUsersSimulation.scala | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 227 +++++++++++++++
.../apache/usergrid/rest/SystemResource.java | 151 +---------
.../apache/usergrid/rest/AbstractRestIT.java | 2 +-
.../apache/usergrid/rest/IndexResourceIT.java | 80 ++++++
22 files changed, 896 insertions(+), 403 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index a6e971a,0000000..23097c4
mode 100644,000000..100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@@ -1,84 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.corepersistence.events;
+
++import com.google.inject.Inject;
++import java.util.List;
++import java.util.UUID;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
++import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
- import java.util.List;
-
- import com.google.inject.Inject;
- import org.apache.usergrid.persistence.EntityManagerFactory;
-
+
+/**
+ * Remove Entity index when specific version of Entity is deleted.
+ * TODO: do we need this? Don't our version-created and entity-deleted handlers take care of this?
+ * If we do need it then it should be wired in via GuiceModule in the corepersistence package.
+ */
+public class EntityVersionDeletedHandler implements EntityVersionDeleted {
+
+ private final SerializationFig serializationFig;
+
+ private EntityManagerFactory emf;
+
+ @Inject
+ public EntityVersionDeletedHandler(SerializationFig fig, EntityManagerFactory emf) {
+ this.serializationFig = fig;
+ }
+
+ @Override
+ public void versionDeleted(
+ final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
+
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ final EntityIndexBatch eibatch = ei.createBatch();
+
+ final IndexScope indexScope = new IndexScopeImpl(
+ new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
+ scope.getName()
+ );
+ rx.Observable.from(entityVersions)
+ .subscribeOn(Schedulers.io())
+ .buffer(serializationFig.getBufferSize())
+ .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ @Override
+ public List<MvccEntity> call(List<MvccEntity> entityList) {
+ for (MvccEntity entity : entityList) {
+ eibatch.deindex(indexScope, entityId, entity.getVersion());
+ }
+ eibatch.execute();
+ return entityList;
+ }
+ }).toBlocking().last();
+ }
++
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
index 65a34e0,aedcf3a..c76be8e
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
@@@ -32,7 -33,7 +32,6 @@@ import org.apache.usergrid.persistence.
*/
public interface EntityVersionDeleted {
--
/**
* The version specified was removed.
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b099678,a3ca8a5..2f555ed
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -64,9 -67,8 +65,9 @@@ import static org.apache.usergrid.persi
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@@ -177,10 -180,9 +179,9 @@@
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ log.debug( "De-indexing type {} with documentId '{}'", entityType, indexId );
- bulkRequest.add( client.prepareDelete(
- indexName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index aa797e2,d546092..86826a5
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -18,13 -18,15 +18,38 @@@
package org.apache.usergrid.persistence.index.impl;
++import com.google.common.collect.ImmutableMap;
++import com.google.inject.Inject;
++import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
--
-import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.utils.StringUtils;
++import org.apache.commons.lang.StringUtils;
++import org.apache.usergrid.persistence.core.scope.ApplicationScope;
++import org.apache.usergrid.persistence.core.util.Health;
++import org.apache.usergrid.persistence.core.util.ValidationUtils;
++import org.apache.usergrid.persistence.index.EntityIndex;
++import org.apache.usergrid.persistence.index.EntityIndexBatch;
++import org.apache.usergrid.persistence.index.IndexFig;
++import org.apache.usergrid.persistence.index.IndexIdentifier;
++import org.apache.usergrid.persistence.index.IndexScope;
++import org.apache.usergrid.persistence.index.SearchTypes;
++import org.apache.usergrid.persistence.index.exceptions.IndexException;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
++import org.apache.usergrid.persistence.index.query.CandidateResult;
++import org.apache.usergrid.persistence.index.query.CandidateResults;
++import org.apache.usergrid.persistence.index.query.Query;
++import org.apache.usergrid.persistence.model.entity.Id;
++import org.apache.usergrid.persistence.model.entity.SimpleId;
++import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@@ -59,32 -59,26 +87,6 @@@ import org.elasticsearch.search.sort.So
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
--import org.apache.usergrid.persistence.core.scope.ApplicationScope;
--import org.apache.usergrid.persistence.core.util.Health;
--import org.apache.usergrid.persistence.core.util.ValidationUtils;
- import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexFig;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.SearchTypes;
--import org.apache.usergrid.persistence.index.exceptions.IndexException;
--import org.apache.usergrid.persistence.index.query.CandidateResult;
--import org.apache.usergrid.persistence.index.query.CandidateResults;
--import org.apache.usergrid.persistence.index.query.Query;
--import org.apache.usergrid.persistence.model.entity.Id;
--import org.apache.usergrid.persistence.model.entity.SimpleId;
--import org.apache.usergrid.persistence.model.util.UUIDGenerator;
--
--import com.google.common.collect.ImmutableMap;
--import com.google.inject.Inject;
--import com.google.inject.assistedinject.Assisted;
--
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
--
/**
* Implements index using ElasticSearch Java API.
@@@ -125,11 -120,8 +128,10 @@@ public class EsEntityIndexImpl implemen
@Inject
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
- final EsProvider provider ) {
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider ) {
++ public EsEntityIndexImpl(
++ @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider ) {
+
ValidationUtils.validateApplicationScope( appScope );
-
this.applicationScope = appScope;
this.esProvider = provider;
this.config = config;
@@@ -147,34 -147,54 +157,63 @@@
createMappings();
}
- final AdminClient admin = esProvider.getClient().admin();
+ //get index name with suffix attached
+ String indexName = indexIdentifier.getIndex(normalizedSuffix);
+
+ //Create index
+ try {
+ final AdminClient admin = esProvider.getClient().admin();
+ Settings settings = ImmutableSettings.settingsBuilder().put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", numberOfReplicas).build();
+ final CreateIndexResponse cir = admin.indices().prepareCreate(indexName).setSettings(settings).execute().actionGet();
+ logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
+ } catch (IndexAlreadyExistsException e) {
+ logger.info("Index Name [{}] already exists", indexName);
+ }
+
+ addAlias(normalizedSuffix);
- final int numberOfShards = config.getNumberOfShards();
- final int numberOfReplicas = config.getNumberOfReplicas();
+ testNewIndex();
+ } catch (IndexAlreadyExistsException expected) {
+ // this is expected to happen if index already exists, it's a no-op and swallow
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to initialize index", e);
+ }
+ }
- Settings settings = ImmutableSettings.settingsBuilder()
- .put( "index.number_of_shards", numberOfShards )
- .put( "index.number_of_replicas", numberOfReplicas ).build();
+ @Override
+ public void addAlias(final String indexSuffix) {
+ try {
+ Boolean isAck;
+ String indexName = indexIdentifier.getIndex(indexSuffix);
+ final AdminClient adminClient = esProvider.getClient().admin();
+
- final CreateIndexResponse cir = admin.indices().prepareCreate( indexName )
- .setSettings( settings ).execute().actionGet();
+ //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
++ ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices()
++ .getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
+
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ String[] indexNames = aliasMap.keys().toArray(String.class);
+ for(String currentIndex : indexNames){
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
++ isAck = adminClient.indices().prepareAliases().removeAlias(
++ currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
++ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",
++ currentIndex, alias, isAck);
- // create the document, this ensures the index is ready
+ }
+
- // Immediately create a document and remove it to ensure the entire cluster is ready
- // to receive documents. Occasionally we see errors.
- // See this post: http://s.apache.org/index-missing-exception
+ //add read alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
++ isAck = adminClient.indices().prepareAliases().addAlias(
++ indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+
- testNewIndex();
- }
- catch ( IndexAlreadyExistsException expected ) {
- // this is expected to happen if index already exists, it's a no-op and swallow
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to initialize index", e );
+ //add write alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
++ isAck = adminClient.indices().prepareAliases().addAlias(
++ indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
+
+ } catch (Exception e) {
+ logger.warn("Failed to create alias ", e);
}
}
@@@ -193,19 -217,18 +236,20 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
- esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
- .get();
++ esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId )
++ .setSource( DEFAULT_PAYLOAD ).get();
- logger.info( "Successfully created new document with docId {} in index {} type {}",
- tempId, indexName, VERIFY_TYPE );
- logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- alias, VERIFY_TYPE );
++ logger.info( "Created new document with docId {} in index {} and type {}",
++ tempId, alias, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName )
- .setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setTypes(VERIFY_TYPE)
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
++ esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() )
++ .setTypes(VERIFY_TYPE)
++ .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+
- logger.info( "Successfully deleted all documents in index {} and type {}", alias, VERIFY_TYPE );
++ logger.info( "Deleted all documents in index {} and type {}",
++ alias, VERIFY_TYPE );
- logger.info( "Successfully deleted all documents in index {} and type {}",
- indexName, VERIFY_TYPE );
return true;
}
@@@ -221,16 -244,16 +265,16 @@@
*/
private void createMappings() throws IOException {
- XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_" );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
+ XContentFactory.jsonBuilder(), "_default_");
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
- .preparePutTemplate( "usergrid_template" )
+ .preparePutTemplate("usergrid_template")
// set mapping as the default for all types
- .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", xcb )
+ .setTemplate(config.getIndexPrefix() + "*").addMapping( "_default_", xcb )
.execute().actionGet();
- if(!pitr.isAcknowledged()){
+ if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
}
}
@@@ -256,9 -279,10 +300,9 @@@
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( entityTypes )
- .setScroll( cursorTimeout + "m" ).setQuery( qb );
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(entityTypes)
- .setScroll(cursorTimeout + "m").setQuery(qb);
-
+
++ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
++ .setTypes(entityTypes).setScroll(cursorTimeout + "m").setQuery(qb);
final FilterBuilder fb = query.createFilterBuilder();
@@@ -307,9 -331,12 +351,11 @@@
}
-
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
- new Object[] { this.indexName, context, entityTypes, srb } );
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
+ this.alias, context, entityTypes, srb
+ } );
+
}
@@@ -397,9 -424,9 +443,10 @@@
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName )
- .execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
- esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
++ esProvider.getClient().admin().indices()
++ .prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
+ logger.debug( "Refreshed index: " + alias);
+
return true;
}
catch ( IndexMissingException e ) {
@@@ -430,15 -457,17 +477,15 @@@
//since we don't have paging inputs, there's no point in executing a query for paging.
- final String context = IndexingUtils.createContextName( scope );
+ final String context = IndexingUtils.createContextName(scope);
final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery(
- IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
-
- final SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
-
++ final QueryBuilder queryBuilder =
++ QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( searchTypes.getTypeNames() ).setScroll( cursorTimeout + "m" )
- .setQuery( queryBuilder );
++ final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
++ .setTypes(searchTypes.getTypeNames())
++ .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
final SearchResponse searchResponse;
try {
@@@ -457,73 -486,17 +504,80 @@@
}
+ @Override
+ public void deleteAllVersionsOfEntity( Id entityId ) {
+
- final TermQueryBuilder tqb =
- QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
++ final TermQueryBuilder tqb = QueryBuilders.termQuery(
++ ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
+
- final DeleteByQueryResponse response =
- esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( tqb ).execute().actionGet();
++ final DeleteByQueryResponse response = esProvider.getClient()
++ .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute().actionGet();
+
+
+ logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() } );
++ new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() });
+
+ checkDeleteByQueryResponse( tqb, response );
+
+ }
+
+
+ @Override
+ public void deletePreviousVersions( final Id id, final UUID version ) {
+
+ final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery( IndexingUtils.ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase() ),
- FilterBuilders.rangeFilter( IndexingUtils.ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
++ QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase()),
++ FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
+
- final DeleteByQueryResponse response =
- esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
++ final DeleteByQueryResponse response = esProvider.getClient()
++ .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( fqb ).execute().actionGet();
+
+ //error message needs to be retooled so that it describes the entity more throughly
- logger.debug( "Deleted entity {}:{} with version {} from all index scopes with response status = {}",
- new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
++ logger.debug( "Deleted entity {}:{} with version {} from all "
++ + "index scopes with response status = {}",
++ new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
+
+ checkDeleteByQueryResponse( fqb, response );
+ }
+
+
+ /**
+ * Validate the response doens't contain errors, if it does, fail fast at the first error we encounter
- * @param query
- * @param response
+ */
- private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {
++ private void checkDeleteByQueryResponse(
++ final QueryBuilder query, final DeleteByQueryResponse response ) {
++
+ for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
+ final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
+
+ for ( ShardOperationFailedException failedException : failures ) {
- throw new IndexException( String.format(
- "Unable to delete by query %s. Failed with code %d and reason %s on shard %s in index %s",
- query.toString(), failedException.status(), failedException.reason(), failedException.shardId(),
++ throw new IndexException( String.format("Unable to delete by query %s. "
++ + "Failed with code %d and reason %s on shard %s in index %s",
++ query.toString(),
++ failedException.status(),
++ failedException.reason(),
++ failedException.shardId(),
+ failedException.index() ) );
+ }
+
+ }
+ }
+
+
/**
* For testing only.
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexIdentifier.getIndex(null) ).get();
++
++ DeleteIndexResponse response = adminClient.indices()
++ .prepareDelete( indexIdentifier.getIndex(null) ).get();
++
if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + alias);
}
else {
- logger.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + alias);
}
}
@@@ -582,7 -555,8 +636,7 @@@
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health( new ClusterHealthRequest( new String[] { indexName } ) ).get();
- .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)}))
- .get();
++ .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)})).get();
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------