You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/12/10 01:17:37 UTC
[01/13] incubator-usergrid git commit: fix deletes so we remove from
all
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o a63b856e9 -> 04679c82a
fix deletes so we remove from all
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4b90e97e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4b90e97e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4b90e97e
Branch: refs/heads/two-dot-o
Commit: 4b90e97e16253559d51479dd57a59aacf3f61466
Parents: c4973e9
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 08:40:54 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 08:40:54 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/EntityIndex.java | 7 ++++
.../index/impl/EsEntityIndexBatchImpl.java | 24 ++++++++----
.../index/impl/EsEntityIndexImpl.java | 18 ++++++---
.../persistence/index/impl/EntityIndexTest.java | 41 +++++++++++++++++++-
4 files changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 4a653b3..6e7779f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -47,6 +47,13 @@ public interface EntityIndex {
public void addIndex(final String indexSuffix, final int shards, final int replicas);
/**
+ * Get the indexes for an alias
+ * @param aliasName name of alias
+ * @return list of index names
+ */
+ public String[] getIndexes(final String aliasName);
+
+ /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index a3ca8a5..35c89ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -26,7 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.*;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -36,9 +36,6 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -59,6 +56,9 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import com.google.common.base.Joiner;
+import rx.Observable;
+import rx.functions.Action1;
+import rx.schedulers.Schedulers;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -92,13 +92,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final FailureMonitor failureMonitor;
+ private final EntityIndex entityIndex;
+
public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client client,
- final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor ) {
+ final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor, final EntityIndex entityIndex ) {
this.applicationScope = applicationScope;
this.client = client;
this.failureMonitor = failureMonitor;
+ this.entityIndex = entityIndex;
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
this.alias = indexIdentifier.getAlias();
this.refresh = config.isForcedRefresh();
@@ -162,7 +165,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
final String context = createContextName( indexScope );
final String entityType = id.getType();
- String indexId = createIndexDocId( id, version, context );
+ final String indexId = createIndexDocId( id, version, context );
if ( log.isDebugEnabled() ) {
@@ -181,8 +184,15 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ final String[] indexes = entityIndex.getIndexes(alias.getReadAlias());
+ //get all indexes then flush everyone
+ Observable.from(indexes).subscribeOn(Schedulers.io()).forEach(new Action1<String>() {
+ @Override
+ public void call(String index) {
+ bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
- bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
+ }
+ });
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d546092..13d07d6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -178,9 +178,9 @@ public class EsEntityIndexImpl implements EntityIndex {
Boolean isAck;
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
- String[] indexNames = aliasMap.keys().toArray(String.class);
+ final String aliasName = alias.getWriteAlias();
+ String[] indexNames = getIndexes( aliasName);
+
for(String currentIndex : indexNames){
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
@@ -198,6 +198,14 @@ public class EsEntityIndexImpl implements EntityIndex {
}
}
+ @Override
+ public String[] getIndexes(String aliasName) {
+ final AdminClient adminClient = esProvider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
+ return aliasMap.keys().toArray(String.class);
+ }
+
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@ -262,7 +270,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public EntityIndexBatch createBatch() {
return new EsEntityIndexBatchImpl(
- applicationScope, esProvider.getClient(), config, 1000, failureMonitor );
+ applicationScope, esProvider.getClient(), config, 1000, failureMonitor, this );
}
@@ -424,7 +432,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
+ esProvider.getClient().admin().indices().prepareRefresh( alias.getReadAlias() ).execute().actionGet();
logger.debug( "Refreshed index: " + alias);
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 13d1552..ced682b 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
@@ -139,6 +140,41 @@ public class EntityIndexTest extends BaseIT {
testQuery(indexScope, searchTypes, entityIndex, "name = 'Lowe Kelley'", 1 );
}
+ @Test
+ public void testDeleteByQueryWithAlias() throws IOException {
+ Id appId = new SimpleId( "application" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ EntityIndex entityIndex = eif.createEntityIndex(applicationScope);
+ entityIndex.initializeIndex();
+
+ final String entityType = "thing";
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+ entityIndex.refresh();
+
+ entityIndex.addIndex("v2", 1,0);
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+ entityIndex.refresh();
+ CandidateResults crs = testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 2);
+
+ EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
+ entityIndexBatch.deindex(indexScope, crs.get(0));
+ entityIndexBatch.deindex(indexScope,crs.get(1));
+ entityIndexBatch.executeAndRefresh();
+ entityIndex.refresh();
+
+ //Hilda Youn
+ testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
+
+ }
+
private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope, String filePath,final int max,final int startIndex) throws IOException {
InputStream is = this.getClass().getResourceAsStream( filePath );
ObjectMapper mapper = new ObjectMapper();
@@ -172,7 +208,7 @@ public class EntityIndexTest extends BaseIT {
- if ( count++ > max ) {
+ if ( ++count > max ) {
break;
}
}
@@ -221,7 +257,7 @@ public class EntityIndexTest extends BaseIT {
}
- private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
+ private CandidateResults testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
StopWatch timer = new StopWatch();
timer.start();
@@ -232,6 +268,7 @@ public class EntityIndexTest extends BaseIT {
assertEquals( num, candidateResults.size() );
log.debug( "Query time {}ms", timer.getTime() );
+ return candidateResults;
}
[09/13] incubator-usergrid git commit: add caching to indexes
Posted by to...@apache.org.
add caching to indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b545c0e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b545c0e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b545c0e4
Branch: refs/heads/two-dot-o
Commit: b545c0e4715d458f7e6b5c1cfa0e96a1f86068b2
Parents: 8d34426
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 9 09:38:45 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 9 09:38:45 2014 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 30 ++-----
.../persistence/index/impl/EsIndexCache.java | 91 ++++++++++++++++++++
2 files changed, 99 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b545c0e4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 3f895b5..a6e9337 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -30,7 +30,6 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -38,8 +37,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -120,11 +117,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
+ private EsIndexCache aliasCache;
- @Inject
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
- final EsProvider provider ) {
+ @Inject
+ public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider, final EsIndexCache indexCache) {
ValidationUtils.validateApplicationScope( appScope );
this.applicationScope = appScope;
this.esProvider = provider;
@@ -133,6 +130,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
+ this.aliasCache = indexCache;
}
@Override
@@ -188,7 +186,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
- String[] indexNames = getIndexes(alias.getWriteAlias());
+ String[] indexNames = getIndexes(AliasType.Write);
for(String currentIndex : indexNames){
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
@@ -206,7 +204,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
-
+ aliasCache.invalidate(alias);
} catch (Exception e) {
logger.warn("Failed to create alias ", e);
}
@@ -214,21 +212,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public String[] getIndexes(final AliasType aliasType) {
- final String aliasName = aliasType == AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
- return getIndexes(aliasName);
+ return aliasCache.getIndexes(alias,aliasType);
}
- /**
- * get indexes for alias
- * @param aliasName
- * @return
- */
- private String[] getIndexes(final String aliasName){
- final AdminClient adminClient = esProvider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
- return aliasMap.keys().toArray(String.class);
- }
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
@@ -457,7 +443,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public boolean doOp() {
try {
- String[] indexes = getIndexes(AliasType.Read);
+ String[] indexes = getIndexes(AliasType.Read);
Observable.from(indexes).subscribeOn(Schedulers.io()).map(new Func1<String, String>() {
@Override
public String call(String index) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b545c0e4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
new file mode 100644
index 0000000..710cc60
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -0,0 +1,91 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Classy class class.
+ */
+@Singleton
+public class EsIndexCache {
+
+ private static final Logger logger = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+
+ private LoadingCache<String, String[]> aliasIndexCache;
+
+ @Inject
+ public EsIndexCache(final EsProvider provider) {
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize(1000)
+ .refreshAfterWrite(5,TimeUnit.MINUTES)
+ .build(new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload(String key, String[] oldValue) throws Exception {
+ return super.reload(key, oldValue);
+ }
+
+ @Override
+ public String[] load(String aliasName) {
+ final AdminClient adminClient = provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
+ return aliasMap.keys().toArray(String.class);
+ }
+ })
+
+ ;
+ }
+
+ public String[] getIndexes(IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ String[] indexes;
+ try {
+ indexes = aliasIndexCache.get(aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias());
+ } catch (ExecutionException ee) {
+ logger.error("Failed to retreive indexes", ee);
+ throw new RuntimeException(ee);
+ }
+ return indexes;
+ }
+
+ public void invalidate(IndexIdentifier.IndexAlias alias){
+ aliasIndexCache.invalidate(alias.getWriteAlias());
+ aliasIndexCache.invalidate(alias.getReadAlias());
+
+ }
+
+}
[12/13] incubator-usergrid git commit: move method back to entityindex
Posted by to...@apache.org.
move method back to entityindex
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/18197e3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/18197e3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/18197e3d
Branch: refs/heads/two-dot-o
Commit: 18197e3de65c29db764218534f846d825bcb4b0f
Parents: 574fe64
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 9 16:02:14 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 9 16:02:14 2014 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityManagerFactory.java | 4 +---
.../apache/usergrid/persistence/index/AliasedEntityIndex.java | 7 -------
.../org/apache/usergrid/persistence/index/EntityIndex.java | 7 +++++++
3 files changed, 8 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/18197e3d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 56eb258..7016d3c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -704,9 +704,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas){
EntityIndex entityIndex = getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope(applicationId));
- if(entityIndex instanceof AliasedEntityIndex) {
- ((AliasedEntityIndex)entityIndex).addIndex(indexSuffix, shards, replicas);
- }
+ entityIndex.addIndex(indexSuffix, shards, replicas);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/18197e3d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
index 15a0d45..118b7ad 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
@@ -38,13 +38,6 @@ public interface AliasedEntityIndex extends EntityIndex{
*/
public void addAlias(final String indexSuffix);
- /**
- * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
- * @param indexSuffix index name
- * @param shards
- * @param replicas
- */
- public void addIndex(final String indexSuffix, final int shards, final int replicas);
/**
* type of alias
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/18197e3d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 549f87e..f1da568 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -38,6 +38,13 @@ public interface EntityIndex {
*/
public void initializeIndex();
+ /**
+ * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
+ * @param indexSuffix index name
+ * @param shards
+ * @param replicas
+ */
+ public void addIndex(final String indexSuffix, final int shards, final int replicas);
/**
* Create the index batch.
[04/13] incubator-usergrid git commit: fix tests
Posted by to...@apache.org.
fix tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3d9137e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3d9137e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3d9137e3
Branch: refs/heads/two-dot-o
Commit: 3d9137e34d9080b6ea15de301470bcb7ca704a8f
Parents: 4d04da8
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 09:28:09 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 09:28:09 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EntityIndexTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d9137e3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index ced682b..d00796d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -88,7 +88,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,0);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
entityIndex.refresh();
@@ -122,7 +122,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,0);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
entityIndex.refresh();
@@ -130,7 +130,7 @@ public class EntityIndexTest extends BaseIT {
entityIndex.addIndex("v2", 1,0);
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100);
entityIndex.refresh();
[11/13] incubator-usergrid git commit: add caching to indexes
Posted by to...@apache.org.
add caching to indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/574fe644
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/574fe644
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/574fe644
Branch: refs/heads/two-dot-o
Commit: 574fe6449a96a7f8a8531cdb0c2f81ad7a2643e9
Parents: 1d2697f
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 9 11:16:16 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 9 11:16:16 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 3 +++
.../persistence/index/impl/EsIndexCache.java | 23 ++++++++++++++++----
2 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/574fe644/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index ea0a4f3..d78ed72 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -110,4 +110,7 @@ public interface IndexFig extends GuicyFig {
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
int getFailRefreshCount();
+
+ @Default("2")
+ int getIndexCacheMaxWorkers();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/574fe644/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index f418590..c607222 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -24,9 +24,13 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.client.AdminClient;
@@ -36,7 +40,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
@@ -46,21 +52,30 @@ import java.util.concurrent.TimeUnit;
public class EsIndexCache {
private static final Logger logger = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+ private final ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<String, String[]> aliasIndexCache;
@Inject
- public EsIndexCache(final EsProvider provider) {
+ public EsIndexCache(final EsProvider provider, final IndexFig indexFig) {
+ this.refreshExecutors = MoreExecutors
+ .listeningDecorator(Executors.newScheduledThreadPool(indexFig.getIndexCacheMaxWorkers()));
aliasIndexCache = CacheBuilder.newBuilder().maximumSize(1000)
.refreshAfterWrite(5,TimeUnit.MINUTES)
.build(new CacheLoader<String, String[]>() {
@Override
- public ListenableFuture<String[]> reload(String key, String[] oldValue) throws Exception {
- return super.reload(key, oldValue);
+ public ListenableFuture<String[]> reload(final String key, String[] oldValue) throws Exception {
+ ListenableFutureTask<String[]> task = ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute(task);
+ return task;
}
@Override
- public String[] load(String aliasName) {
+ public String[] load(final String aliasName) {
final AdminClient adminClient = provider.getClient().admin();
//remove write alias, can only have one
ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
[02/13] incubator-usergrid git commit: Add enum to prevent mistakes
in naming
Posted by to...@apache.org.
Add enum to prevent mistakes in naming
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7e347422
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7e347422
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7e347422
Branch: refs/heads/two-dot-o
Commit: 7e3474227a5f97faea141a9b6bdf2246327e2ac0
Parents: 4b90e97
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 08:51:22 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 08:51:22 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/EntityIndex.java | 9 ++++++++-
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../persistence/index/impl/EsEntityIndexImpl.java | 11 +++++++----
3 files changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e347422/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 6e7779f..f38a390 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -51,7 +51,7 @@ public interface EntityIndex {
* @param aliasName name of alias
* @return list of index names
*/
- public String[] getIndexes(final String aliasName);
+ public String[] getIndexes(final AliasType aliasType);
/**
* Create the index batch.
@@ -95,6 +95,13 @@ public interface EntityIndex {
*/
public Health getIndexHealth();
+ /**
+ * type of alias
+ */
+ public enum AliasType {
+ Read, Write
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e347422/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 35c89ae..ecc71e9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -184,7 +184,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- final String[] indexes = entityIndex.getIndexes(alias.getReadAlias());
+ final String[] indexes = entityIndex.getIndexes(EntityIndex.AliasType.Read);
//get all indexes then flush everyone
Observable.from(indexes).subscribeOn(Schedulers.io()).forEach(new Action1<String>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e347422/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 13d07d6..610fe6c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -178,8 +178,7 @@ public class EsEntityIndexImpl implements EntityIndex {
Boolean isAck;
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
- final String aliasName = alias.getWriteAlias();
- String[] indexNames = getIndexes( aliasName);
+ String[] indexNames = getIndexes(alias.getWriteAlias());
for(String currentIndex : indexNames){
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
@@ -199,14 +198,18 @@ public class EsEntityIndexImpl implements EntityIndex {
}
@Override
- public String[] getIndexes(String aliasName) {
+ public String[] getIndexes(final AliasType aliasType) {
+ final String aliasName = aliasType == AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
+ return getIndexes(aliasName);
+ }
+
+ private String[] getIndexes(final String aliasName){
final AdminClient adminClient = esProvider.getClient().admin();
//remove write alias, can only have one
ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
return aliasMap.keys().toArray(String.class);
}
-
/**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
* http://s.apache.org/index-missing-exception
[03/13] incubator-usergrid git commit: get indexes for alias
Posted by to...@apache.org.
get indexes for alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4d04da8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4d04da8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4d04da8e
Branch: refs/heads/two-dot-o
Commit: 4d04da8e58eff556e2f1aab1dbdc9b6c7eb1f740
Parents: 7e34742
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 08:51:47 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 08:51:47 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4d04da8e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 610fe6c..5824c38 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -203,6 +203,11 @@ public class EsEntityIndexImpl implements EntityIndex {
return getIndexes(aliasName);
}
+ /**
+ * get indexes for alias
+ * @param aliasName
+ * @return
+ */
private String[] getIndexes(final String aliasName){
final AdminClient adminClient = esProvider.getClient().admin();
//remove write alias, can only have one
[06/13] incubator-usergrid git commit: add comment
Posted by to...@apache.org.
add comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8cdfaf95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8cdfaf95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8cdfaf95
Branch: refs/heads/two-dot-o
Commit: 8cdfaf9547da2a69a34b9b548167201c6a1a3068
Parents: 983b20b
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 10:14:55 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 10:14:55 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8cdfaf95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 51fe63b..f2cbc6b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -187,6 +187,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
String[] indexes = entityIndex.getIndexes(EntityIndex.AliasType.Read);
+ //get the default index if no alias exists yet
if(indexes == null ||indexes.length == 0){
indexes = new String[]{indexIdentifier.getIndex(null)};
}
[13/13] incubator-usergrid git commit: Removed subscribeOnCalls
Posted by to...@apache.org.
Removed subscribeOnCalls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/04679c82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/04679c82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/04679c82
Branch: refs/heads/two-dot-o
Commit: 04679c82a5a5cac1427c9775d40f3f3d76f30bff
Parents: 18197e3
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Dec 9 17:17:12 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Dec 9 17:17:12 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/04679c82/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index cd5be7f..99a08da 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -192,7 +192,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
indexes = new String[]{indexIdentifier.getIndex(null)};
}
//get all indexes then flush everyone
- Observable.from(indexes).subscribeOn(Schedulers.io())
+ Observable.from(indexes)
.map(new Func1<String, Object>() {
@Override
public Object call(String index) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/04679c82/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a6e9337..67e1680 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -444,7 +444,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public boolean doOp() {
try {
String[] indexes = getIndexes(AliasType.Read);
- Observable.from(indexes).subscribeOn(Schedulers.io()).map(new Func1<String, String>() {
+ Observable.from(indexes).map(new Func1<String, String>() {
@Override
public String call(String index) {
esProvider.getClient().admin().indices().prepareRefresh( index ).execute().actionGet();
[07/13] incubator-usergrid git commit: merge
Posted by to...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/21630f49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/21630f49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/21630f49
Branch: refs/heads/two-dot-o
Commit: 21630f498e7f59cd760e0fd2cd69c1f38de25f45
Parents: 8cdfaf9 a63b856
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 17:23:52 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 17:23:52 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 230 ++++++++++---------
.../corepersistence/CpEntityManagerFactory.java | 6 +-
.../corepersistence/util/CpEntityMapUtils.java | 32 ++-
.../batch/job/SchedulerRuntimeIntervalIT.java | 49 ++--
.../migration/EntityDataMigrationIT.java | 139 +++++------
.../migration/EntityTypeMappingMigrationIT.java | 89 +++----
.../apache/usergrid/persistence/CounterIT.java | 89 ++++---
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityDictionaryIT.java | 11 +-
.../usergrid/persistence/EntityManagerIT.java | 47 ++--
.../persistence/GeoQueryBooleanTest.java | 7 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/PathQueryIT.java | 10 +-
.../PerformanceEntityRebuildIndexTest.java | 6 +-
.../usergrid/persistence/PermissionsIT.java | 5 +-
.../cassandra/EntityManagerFactoryImplIT.java | 9 +-
.../query/AbstractIteratingQueryIT.java | 3 +-
stack/core/src/test/resources/log4j.properties | 5 +-
.../data/DataMigrationManagerImpl.java | 29 ++-
.../persistence/index/AliasedEntityIndex.java | 55 +++++
.../usergrid/persistence/index/EntityIndex.java | 26 ---
.../persistence/index/IndexIdentifier.java | 4 +
.../index/impl/EsEntityIndexBatchImpl.java | 6 +-
.../index/impl/EsEntityIndexImpl.java | 88 ++++---
.../persistence/index/impl/EntityIndexTest.java | 15 +-
.../persistence/queue/DefaultQueueManager.java | 68 ++++++
.../usergrid/services/TestQueueManager.java | 65 ------
.../AbstractServiceNotificationIT.java | 16 +-
.../apns/NotificationsServiceIT.java | 12 +-
.../gcm/NotificationsServiceIT.java | 15 +-
30 files changed, 647 insertions(+), 507 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f6b0df8,f6b0df8..56eb258
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -64,6 -64,6 +64,7 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
++import org.apache.usergrid.persistence.index.AliasedEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.query.Query;
@@@ -702,7 -702,7 +703,10 @@@ public class CpEntityManagerFactory imp
@Override
public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas){
-- getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope( applicationId )).addIndex(indexSuffix,shards,replicas);
++ EntityIndex entityIndex = getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope(applicationId));
++ if(entityIndex instanceof AliasedEntityIndex) {
++ ((AliasedEntityIndex)entityIndex).addIndex(indexSuffix, shards, replicas);
++ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
index 0000000,0000000..15a0d45
new file mode 100644
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/AliasedEntityIndex.java
@@@ -1,0 -1,0 +1,55 @@@
++/*
++ *
++ * * Licensed to the Apache Software Foundation (ASF) under one or more
++ * * contributor license agreements. The ASF licenses this file to You
++ * * under the Apache License, Version 2.0 (the "License"); you may not
++ * * use this file except in compliance with the License.
++ * * You may obtain a copy of the License at
++ * *
++ * * http://www.apache.org/licenses/LICENSE-2.0
++ * *
++ * * Unless required by applicable law or agreed to in writing, software
++ * * distributed under the License is distributed on an "AS IS" BASIS,
++ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * * See the License for the specific language governing permissions and
++ * * limitations under the License. For additional information regarding
++ * * copyright in this work, please see the NOTICE file in the top level
++ * * directory of this distribution.
++ *
++ */
++
++package org.apache.usergrid.persistence.index;
++
++/**
++ * EntityIndex with aliases for multiple indexes
++ */
++public interface AliasedEntityIndex extends EntityIndex{
++
++ /**
++ * Get the indexes for an alias
++ * @param aliasType name of alias
++ * @return list of index names
++ */
++ public String[] getIndexes(final AliasType aliasType);
++
++ /**
++ * Add alias to index, will remove old index from write alias
++ * @param indexSuffix must be different than current index
++ */
++ public void addAlias(final String indexSuffix);
++
++ /**
++ * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
++ * @param indexSuffix index name
++ * @param shards
++ * @param replicas
++ */
++ public void addIndex(final String indexSuffix, final int shards, final int replicas);
++
++ /**
++ * type of alias
++ */
++ public enum AliasType {
++ Read, Write
++ }
++}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index f38a390,4a653b3..549f87e
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@@ -38,31 -38,24 +38,12 @@@ public interface EntityIndex
*/
public void initializeIndex();
-- /**
-- * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
-- * @param indexSuffix index name
-- * @param shards
-- * @param replicas
-- */
-- public void addIndex(final String indexSuffix, final int shards, final int replicas);
-
- /**
- * Get the indexes for an alias
- * @param aliasName name of alias
- * @return list of index names
- */
- public String[] getIndexes(final AliasType aliasType);
/**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
-- /**
-- * Add alias to index, will remove old index from write alias
-- * @param indexSuffix must be different than current index
-- */
-- public void addAlias(final String indexSuffix);
/**
* Execute query in Usergrid syntax.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index f2cbc6b,a3ca8a5..0d0e52b
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -94,11 -92,9 +94,11 @@@ public class EsEntityIndexBatchImpl imp
private final FailureMonitor failureMonitor;
- private final EntityIndex entityIndex;
++ private final AliasedEntityIndex entityIndex;
+
public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client client,
- final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor, final EntityIndex entityIndex ) {
- final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor ) {
++ final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor, final AliasedEntityIndex entityIndex ) {
this.applicationScope = applicationScope;
this.client = client;
@@@ -186,30 -181,9 +186,30 @@@
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- String[] indexes = entityIndex.getIndexes(EntityIndex.AliasType.Read);
-
- bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
-
++ String[] indexes = entityIndex.getIndexes(AliasedEntityIndex.AliasType.Read);
+ //get the default index if no alias exists yet
+ if(indexes == null ||indexes.length == 0){
+ indexes = new String[]{indexIdentifier.getIndex(null)};
+ }
+ final AtomicInteger errorCount = new AtomicInteger();
+ //get all indexes then flush everyone
+ Observable.from(indexes).subscribeOn(Schedulers.io())
+ .map(new Func1<String, Object>() {
+ @Override
+ public Object call(String index) {
+ try {
+ bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
+ }catch (Exception e){
+ log.error("failed to deindex",e);
+ errorCount.incrementAndGet();
+ }
+ return index;
+ }
+ }).toBlocking().last();
+
+ if(errorCount.get()>0){
+ log.error("Failed to flush some indexes");
+ }
log.debug( "Deindexed Entity with index id " + indexId );
maybeFlush();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 5824c38,eb050d6..4bca8c2
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -73,6 -72,6 +72,9 @@@ import org.apache.usergrid.persistence.
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
++import rx.Observable;
++import rx.functions.Func1;
++import rx.schedulers.Schedulers;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
@@@ -83,7 -82,7 +85,7 @@@ import static org.apache.usergrid.persi
/**
* Implements index using ElasticSearch Java API.
*/
--public class EsEntityIndexImpl implements EntityIndex {
++public class EsEntityIndexImpl implements AliasedEntityIndex {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
@@@ -178,18 -184,27 +187,24 @@@
Boolean isAck;
String indexName = indexIdentifier.getIndex(indexSuffix);
final AdminClient adminClient = esProvider.getClient().admin();
+
- //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices()
- .getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
- String[] indexNames = aliasMap.keys().toArray(String.class);
+ String[] indexNames = getIndexes(alias.getWriteAlias());
for(String currentIndex : indexNames){
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
+ isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
+ alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
}
+
//add read alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ isAck = adminClient.indices().prepareAliases().addAlias(
+ indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+
//add write alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ isAck = adminClient.indices().prepareAliases().addAlias(
+ indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
} catch (Exception e) {
@@@ -233,17 -231,19 +248,19 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
- .get();
+ esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
++ .setSource(DEFAULT_PAYLOAD).get();
- logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- alias, VERIFY_TYPE );
+ logger.info( "Successfully created new document with docId {} "
+ + "in index {} and type {}", tempId, alias, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setTypes(VERIFY_TYPE)
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+ esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() )
+ .setTypes(VERIFY_TYPE)
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
++ .setQuery(MATCH_ALL_QUERY_BUILDER).get();
- logger.info( "Successfully deleted all documents in index {} and type {}", alias, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}",
+ alias, VERIFY_TYPE );
return true;
@@@ -440,7 -440,8 +457,15 @@@
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( alias.getReadAlias() ).execute().actionGet();
- esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() )
- .execute().actionGet();
++ String[] indexes = getIndexes(AliasType.Read);
++ Observable.from(indexes).subscribeOn(Schedulers.io()).map(new Func1<String, String>() {
++ @Override
++ public String call(String index) {
++ esProvider.getClient().admin().indices().prepareRefresh( index ).execute().actionGet();
++ return index;
++ }
++ });
++
logger.debug( "Refreshed index: " + alias);
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/21630f49/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 47c4dbd,13d1552..fd94985
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@@ -25,7 -25,6 +25,8 @@@ import java.util.LinkedHashMap
import java.util.List;
import java.util.Map;
++import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
@@@ -38,11 -37,11 +39,6 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.core.util.Health;
--import org.apache.usergrid.persistence.index.EntityIndex;
--import org.apache.usergrid.persistence.index.EntityIndexBatch;
--import org.apache.usergrid.persistence.index.EntityIndexFactory;
--import org.apache.usergrid.persistence.index.IndexScope;
--import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
@@@ -115,7 -114,7 +111,7 @@@ public class EntityIndexTest extends Ba
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-- EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
++ AliasedEntityIndex entityIndex =(AliasedEntityIndex) eif.createEntityIndex( applicationScope );
entityIndex.initializeIndex();
final String entityType = "thing";
@@@ -140,41 -139,6 +136,42 @@@
testQuery(indexScope, searchTypes, entityIndex, "name = 'Lowe Kelley'", 1 );
}
+ @Test
+ public void testDeleteByQueryWithAlias() throws IOException {
+ Id appId = new SimpleId( "application" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
- EntityIndex entityIndex = eif.createEntityIndex(applicationScope);
++ AliasedEntityIndex entityIndex =(AliasedEntityIndex) eif.createEntityIndex( applicationScope );
++
+ entityIndex.initializeIndex();
+
+ final String entityType = "thing";
+ IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+ final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+ entityIndex.refresh();
+
- entityIndex.addIndex("v2", 1,0);
++ entityIndex.addIndex("v2", 1, 0);
+
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
++ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 1, 0);
+
+ entityIndex.refresh();
+ CandidateResults crs = testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 2);
+
+ EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
+ entityIndexBatch.deindex(indexScope, crs.get(0));
+ entityIndexBatch.deindex(indexScope, crs.get(1));
+ entityIndexBatch.executeAndRefresh();
+ entityIndex.refresh();
+
+ //Hilda Youn
+ testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
+
+ }
+
private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope, String filePath,final int max,final int startIndex) throws IOException {
InputStream is = this.getClass().getResourceAsStream( filePath );
ObjectMapper mapper = new ObjectMapper();
[05/13] incubator-usergrid git commit: if there is no alias then use
the index
Posted by to...@apache.org.
if there is no alias then use the index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/983b20b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/983b20b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/983b20b2
Branch: refs/heads/two-dot-o
Commit: 983b20b2179f1363eebaf8cce5771d5d732b2409
Parents: 3d9137e
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 10:11:11 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 10:11:11 2014 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 33 ++++++++++++++------
.../persistence/index/impl/EntityIndexTest.java | 7 +++--
2 files changed, 28 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/983b20b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index ecc71e9..51fe63b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.usergrid.persistence.index.*;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -58,6 +59,7 @@ import org.apache.usergrid.persistence.model.field.value.EntityObject;
import com.google.common.base.Joiner;
import rx.Observable;
import rx.functions.Action1;
+import rx.functions.Func1;
import rx.schedulers.Schedulers;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
@@ -184,16 +186,29 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
- final String[] indexes = entityIndex.getIndexes(EntityIndex.AliasType.Read);
+ String[] indexes = entityIndex.getIndexes(EntityIndex.AliasType.Read);
+ if(indexes == null ||indexes.length == 0){
+ indexes = new String[]{indexIdentifier.getIndex(null)};
+ }
+ final AtomicInteger errorCount = new AtomicInteger();
//get all indexes then flush everyone
- Observable.from(indexes).subscribeOn(Schedulers.io()).forEach(new Action1<String>() {
- @Override
- public void call(String index) {
- bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
-
- }
- });
-
+ Observable.from(indexes).subscribeOn(Schedulers.io())
+ .map(new Func1<String, Object>() {
+ @Override
+ public Object call(String index) {
+ try {
+ bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
+ }catch (Exception e){
+ log.error("failed to deindex",e);
+ errorCount.incrementAndGet();
+ }
+ return index;
+ }
+ }).toBlocking().last();
+
+ if(errorCount.get()>0){
+ log.error("Failed to flush some indexes");
+ }
log.debug( "Deindexed Entity with index id " + indexId );
maybeFlush();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/983b20b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index d00796d..47c4dbd 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -166,7 +166,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
entityIndexBatch.deindex(indexScope, crs.get(0));
- entityIndexBatch.deindex(indexScope,crs.get(1));
+ entityIndexBatch.deindex(indexScope, crs.get(1));
entityIndexBatch.executeAndRefresh();
entityIndex.refresh();
@@ -248,8 +248,9 @@ public class EntityIndexTest extends BaseIT {
Query.fromQL( "name contains 'Ferrari*'" ) );
assertEquals( 1, candidateResults.size() );
- entityIndex.createBatch().deindex( indexScope, entity ).execute();
-
+ EntityIndexBatch batch = entityIndex.createBatch();
+ batch.deindex(indexScope, entity).execute();
+ batch.executeAndRefresh();
entityIndex.refresh();
candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), Query.fromQL( "name contains 'Ferrari*'" ) );
[08/13] incubator-usergrid git commit: add observable to refresh
Posted by to...@apache.org.
add observable to refresh
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8d344268
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8d344268
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8d344268
Branch: refs/heads/two-dot-o
Commit: 8d344268ba435253497a7c92db895152c193a20a
Parents: 21630f4
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 8 17:32:03 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 8 17:32:03 2014 -0700
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 6 +-----
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 2 +-
2 files changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d344268/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 0d0e52b..cd5be7f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -191,7 +191,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if(indexes == null ||indexes.length == 0){
indexes = new String[]{indexIdentifier.getIndex(null)};
}
- final AtomicInteger errorCount = new AtomicInteger();
//get all indexes then flush everyone
Observable.from(indexes).subscribeOn(Schedulers.io())
.map(new Func1<String, Object>() {
@@ -201,15 +200,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
}catch (Exception e){
log.error("failed to deindex",e);
- errorCount.incrementAndGet();
+ throw e;
}
return index;
}
}).toBlocking().last();
- if(errorCount.get()>0){
- log.error("Failed to flush some indexes");
- }
log.debug( "Deindexed Entity with index id " + indexId );
maybeFlush();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d344268/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 4bca8c2..3f895b5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -464,7 +464,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
esProvider.getClient().admin().indices().prepareRefresh( index ).execute().actionGet();
return index;
}
- });
+ }).toBlocking().last();
logger.debug( "Refreshed index: " + alias);
[10/13] incubator-usergrid git commit: add caching to indexes
Posted by to...@apache.org.
add caching to indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1d2697fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1d2697fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1d2697fc
Branch: refs/heads/two-dot-o
Commit: 1d2697fc967680aa092b42705ffdc688e3579dae
Parents: b545c0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Dec 9 09:50:45 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Dec 9 09:50:45 2014 -0700
----------------------------------------------------------------------
.../persistence/index/impl/EsIndexCache.java | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1d2697fc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index 710cc60..f418590 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -40,7 +40,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
- * Classy class class.
+ * Cache for Es index operations
*/
@Singleton
public class EsIndexCache {
@@ -66,11 +66,15 @@ public class EsIndexCache {
ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
return aliasMap.keys().toArray(String.class);
}
- })
-
- ;
+ }) ;
}
+ /**
+ * Get indexes for an alias
+ * @param alias
+ * @param aliasType
+ * @return
+ */
public String[] getIndexes(IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
String[] indexes;
try {
@@ -82,6 +86,10 @@ public class EsIndexCache {
return indexes;
}
+ /**
+ * clean up cache
+ * @param alias
+ */
public void invalidate(IndexIdentifier.IndexAlias alias){
aliasIndexCache.invalidate(alias.getWriteAlias());
aliasIndexCache.invalidate(alias.getReadAlias());