You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/12/10 18:14:08 UTC
[19/41] incubator-usergrid git commit: fix deletes so we remove from
all
fix deletes so we remove from all
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4b90e97e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4b90e97e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4b90e97e
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 4b90e97e16253559d51479dd57a59aacf3f61466
Parents: c4973e9
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 08:40:54 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 08:40:54 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/EntityIndex.java | 7 ++++
.../index/impl/EsEntityIndexBatchImpl.java | 24 ++++++++----
.../index/impl/EsEntityIndexImpl.java | 18 ++++++---
.../persistence/index/impl/EntityIndexTest.java | 41 +++++++++++++++++++-
4 files changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 4a653b3..6e7779f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -47,6 +47,13 @@ public interface EntityIndex {
public void addIndex(final String indexSuffix, final int shards, final int replicas);
/**
+ * Get the indexes for an alias
+ * @param aliasName name of alias
+ * @return list of index names
+ */
+ public String[] getIndexes(final String aliasName);
+
+ /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index a3ca8a5..35c89ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.*;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -36,9 +36,6 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -59,6 +56,9 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import com.google.common.base.Joiner;
+import rx.Observable;
+import rx.functions.Action1;
+import rx.schedulers.Schedulers;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -92,13 +92,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final FailureMonitor failureMonitor;
+ private final EntityIndex entityIndex;
+
public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client client,
- final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor ) {
+ final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor, final EntityIndex entityIndex ) {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
+ this.entityIndex = entityIndex;
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
this.alias = indexIdentifier.getAlias();
this.refresh = config.isForcedRefresh();
@@ -162,7 +165,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
final String context = createContextName( indexScope );
final String entityType = id.getType();
- String indexId = createIndexDocId( id, version, context );
+ final String indexId = createIndexDocId( id, version, context );
if ( log.isDebugEnabled() ) {
@@ -181,8 +184,15 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ final String[] indexes = entityIndex.getIndexes(alias.getReadAlias());
+ //get all indexes then flush everyone
+ Observable.from(indexes).subscribeOn(Schedulers.io()).forEach(new Action1<String>() {
+ @Override
+ public void call(String index) {
+ bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
- bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
+ }
+ });
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d546092..13d07d6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -178,9 +178,9 @@ public class EsEntityIndexImpl implements EntityIndex {
Boolean isAck;
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
- String[] indexNames = aliasMap.keys().toArray(String.class);
+ final String aliasName = alias.getWriteAlias();
+ String[] indexNames = getIndexes( aliasName);
+
for(String currentIndex : indexNames){
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
@@ -198,6 +198,14 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
+ @Override
+ public String[] getIndexes(String aliasName) {
+ final AdminClient adminClient = esProvider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
+ return aliasMap.keys().toArray(String.class);
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@ -262,7 +270,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public EntityIndexBatch createBatch() {
return new EsEntityIndexBatchImpl(
- applicationScope, esProvider.getClient(), config, 1000, failureMonitor );
+ applicationScope, esProvider.getClient(), config, 1000, failureMonitor, this );
}
@@ -424,7 +432,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
+ esProvider.getClient().admin().indices().prepareRefresh( alias.getReadAlias() ).execute().actionGet();
logger.debug( "Refreshed index: " + alias);
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 13d1552..ced682b 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
@@ -139,6 +140,41 @@ public class EntityIndexTest extends BaseIT {
testQuery(indexScope, searchTypes, entityIndex, "name = 'Lowe Kelley'", 1 );
}
+ @Test
+ public void testDeleteByQueryWithAlias() throws IOException {
+ Id appId = new SimpleId( "application" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ EntityIndex entityIndex = eif.createEntityIndex(applicationScope);
+ entityIndex.initializeIndex();
+
+ final String entityType = "thing";
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+ entityIndex.refresh();
+
+ entityIndex.addIndex("v2", 1,0);
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+ entityIndex.refresh();
+ CandidateResults crs = testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 2);
+
+ EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
+ entityIndexBatch.deindex(indexScope, crs.get(0));
+ entityIndexBatch.deindex(indexScope,crs.get(1));
+ entityIndexBatch.executeAndRefresh();
+ entityIndex.refresh();
+
+ //Hilda Youn
+ testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
+
+ }
+
private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope, String filePath,final int max,final int startIndex) throws IOException {
InputStream is = this.getClass().getResourceAsStream( filePath );
ObjectMapper mapper = new ObjectMapper();
@@ -172,7 +208,7 @@ public class EntityIndexTest extends BaseIT {
- if ( count++ > max ) {
+ if ( ++count > max ) {
break;
}
}
@@ -221,7 +257,7 @@ public class EntityIndexTest extends BaseIT {
}
- private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
+ private CandidateResults testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
StopWatch timer = new StopWatch();
timer.start();
@@ -232,6 +268,7 @@ public class EntityIndexTest extends BaseIT {
assertEquals( num, candidateResults.size() );
log.debug( "Query time {}ms", timer.getTime() );
+ return candidateResults;
}