You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/03/30 21:52:17 UTC
[01/24] incubator-usergrid git commit: add old type mapping to queries
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev 6db7ce937 -> 07cb9dfd6
add old type mapping to queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cc779b24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cc779b24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cc779b24
Branch: refs/heads/two-dot-o-dev
Commit: cc779b24c6e8d19d6a20da85d2d625d8407b35eb
Parents: 8fe7267
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 23 10:53:20 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 23 10:53:20 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/SearchType.java | 10 ++++++----
.../apache/usergrid/persistence/index/SearchTypes.java | 3 ++-
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 13 ++++++++-----
.../usergrid/persistence/index/impl/IndexingUtils.java | 2 ++
4 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
index 5501c09..c039360 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
@@ -40,10 +40,12 @@ public class SearchType{
return new SearchType( id.getType() );
}
-
-
public String getTypeName(ApplicationScope applicationScope) {
- final String typeName = IndexingUtils.getType(applicationScope, type);
- return typeName;
+ return IndexingUtils.getType(applicationScope, type);
+ }
+
+ public String[] getTypeNames(ApplicationScope applicationScope) {
+ final String[] typeNames = new String[]{type , IndexingUtils.getType(applicationScope, type)};
+ return typeNames;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 1a97da2..9970c62 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -42,10 +42,11 @@ public class SearchTypes {
public String[] getTypeNames(ApplicationScope applicationScope) {
- String[] typeNames = new String[types.length];
+ String[] typeNames = new String[types.length*2];
int i =0 ;
for(String type : types){
typeNames[i++] = IndexingUtils.getType(applicationScope,type);
+ typeNames[i++] = type;
}
return typeNames;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 90b8e35..3b65c74 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -159,8 +159,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
indexes = new String[]{indexIdentifier.getIndex(null)};
}
- container.addDeIndexRequest( new DeIndexRequest( indexes, entityType.getTypeName(applicationScope), indexId ) );
+ String[] typeNames = entityType.getTypeNames(applicationScope);
+ for(String type : typeNames) {
+ container.addDeIndexRequest(new DeIndexRequest(indexes, type, indexId));
+ }
log.debug("Deindexed Entity with index id " + indexId);
return this;
@@ -242,12 +245,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put( ARRAY_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -259,12 +262,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
+ entityMap.put(SET_PREFIX+ field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
EntityObject eo = ( EntityObject ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), entityToMap(eo) ); // recursion
+ entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
}
else if ( f instanceof StringField ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cc779b24/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c365ff4..2fb56d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -40,6 +40,8 @@ public class IndexingUtils {
public static final String BOOLEAN_PREFIX = "bu_";
public static final String ARRAY_PREFIX = "ar_";
+ public static final String SET_PREFIX = "set_";
+ public static final String EO_PREFIX = "eo_";
public static final String SPLITTER = "\\__";
[04/24] incubator-usergrid git commit: factor logic into index request
Posted by to...@apache.org.
factor logic into index request
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e0a79dbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e0a79dbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e0a79dbb
Branch: refs/heads/two-dot-o-dev
Commit: e0a79dbb24e1f9060b34c99effc3b966af4eaa56
Parents: a9eafb8
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 13:58:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 13:58:21 2015 -0600
----------------------------------------------------------------------
.../index/ApplicationEntityIndex.java | 2 --
.../impl/EsApplicationEntityIndexImpl.java | 6 ------
.../persistence/index/impl/IndexRequest.java | 17 +++++++++++------
.../index/impl/BufferQueueSQSImplTest.java | 20 ++++++++++++--------
.../persistence/index/impl/EntityIndexTest.java | 2 +-
5 files changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 4808d8a..dd1c5fa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -19,10 +19,8 @@
*/
package org.apache.usergrid.persistence.index;
-import com.google.common.util.concurrent.ListenableFuture;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
-import org.elasticsearch.action.ListenableActionFuture;
import rx.Observable;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 4c3e42c..1debe3b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -22,11 +22,8 @@ package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -61,12 +58,9 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Action1;
-import rx.schedulers.Schedulers;
import java.util.*;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 23f3d08..9eae481 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -26,6 +26,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
@@ -49,18 +50,22 @@ public class IndexRequest implements BatchRequest {
public Map<String, Object> data;
-
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexScope indexScope, Entity entity) {
- String context = createContextName(applicationScope,indexScope);
- SearchType searchType = SearchType.fromId(entity.getId());
- final Map<String, Object> data = EntityToMapConverter.convert(entity,context);
+ this(writeAlias, applicationScope, createContextName(applicationScope, indexScope), entity);
+ }
+
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, String context , Entity entity) {
+ this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(entity, context));
+ }
+
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType searchType, String documentId, Map<String, Object> data) {
data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
this.entityType = searchType.getTypeName(applicationScope);
- this.documentId = IndexingUtils.createIndexDocId(entity,context);
this.data = data;
- }
+ this.documentId = documentId;
+ }
/**
* DO NOT DELETE! Required for Jackson
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
index 9a362cb..6fb6021 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImplTest.java
@@ -20,12 +20,15 @@
package org.apache.usergrid.persistence.index.impl;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -90,22 +93,23 @@ public class BufferQueueSQSImplTest {
@Test
public void testMessageIndexing(){
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
assumeTrue( ugProvider.getCredentials().getAWSAccessKeyId() != null );
assumeTrue( ugProvider.getCredentials().getAWSSecretKey() != null );
final Map<String, Object> request1Data = new HashMap<String, Object>() {{put("test", "testval1");}};
- final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", "testType1", "testDoc1",request1Data );
+ final IndexRequest indexRequest1 = new IndexRequest( "testAlias1", applicationScope, SearchType.fromType("testType1"), "testDoc1",request1Data );
final Map<String, Object> request2Data = new HashMap<String, Object>() {{put("test", "testval2");}};
- final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", "testType2", "testDoc2",request2Data );
+ final IndexRequest indexRequest2 = new IndexRequest( "testAlias2", applicationScope, SearchType.fromType( "testType2"), "testDoc2",request2Data );
//de-index request
- final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, "testType3", "testId3" );
+ final DeIndexRequest deIndexRequest1 = new DeIndexRequest( new String[]{"index1.1, index1.2"}, applicationScope, new IndexScopeImpl(new SimpleId("testId3"),"name3"), new SimpleId("id3"), UUID.randomUUID() );
- final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, "testType4", "testId4" );
+ final DeIndexRequest deIndexRequest2 = new DeIndexRequest( new String[]{"index2.1", "index2.1"}, applicationScope, new IndexScopeImpl(new SimpleId("testId4"),"name4"), new SimpleId("id4"), UUID.randomUUID() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e0a79dbb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 85570f1..6df5a6d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -651,7 +651,7 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
- final Query query = Query.fromQL( "select * order by created" );
+ Query query = Query.fromQL( "select * order by created" );
query.setLimit( limit );
if ( cursor != null ) {
[18/24] incubator-usergrid git commit: fixed tests
Posted by to...@apache.org.
fixed tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b98cdce0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b98cdce0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b98cdce0
Branch: refs/heads/two-dot-o-dev
Commit: b98cdce0f6511cadd5490b7c79617b0789b7a011
Parents: a7e08db
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:48:48 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:48:48 2015 -0600
----------------------------------------------------------------------
.../persistence/index/EsIndexCache.java | 29 -----
.../usergrid/persistence/index/IndexCache.java | 29 +++++
.../persistence/index/guice/IndexModule.java | 2 +-
.../impl/EsApplicationEntityIndexImpl.java | 5 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 17 +--
.../index/impl/EsIndexCacheImpl.java | 2 +-
.../migration/EsIndexDataMigrationImpl.java | 23 ++--
.../index/guice/TestIndexModule.java | 12 +-
.../index/impl/IndexMigrationTest.java | 112 +++++++++++++++++++
10 files changed, 167 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
deleted file mode 100644
index 6427459..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.persistence.index;
-
-/**
- * Classy class class.
- */
-public interface EsIndexCache {
- String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
-
- void invalidate(IndexAlias alias);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
new file mode 100644
index 0000000..253c66d
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
@@ -0,0 +1,29 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Classy class class.
+ */
+public interface IndexCache {
+ String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+
+ void invalidate(IndexAlias alias);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index bb5ec54..6307382 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -51,7 +51,7 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndexFactory.class).to(EsEntityIndexFactoryImpl.class);
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
- bind(EsIndexCache.class).to(EsIndexCacheImpl.class);
+ bind(IndexCache.class).to(EsIndexCacheImpl.class);
bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 53286b4..9248980 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -37,7 +37,6 @@ import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
@@ -73,7 +72,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
private final IndexBufferProducer indexBatchBufferProducer;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -85,7 +84,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
@Inject
public EsApplicationEntityIndexImpl(@Assisted ApplicationScope appScope, final AliasedEntityIndex entityIndex, final IndexFig config,
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final MapManagerFactory mapManagerFactory, final IndexFig indexFig, final IndexIdentifier indexIdentifier){
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 426f587..78f0e01 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -39,7 +39,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
@@ -57,7 +57,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
} );
@Inject
- public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final IndexCache indexCache,
final IndexBufferProducer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig, final AliasedEntityIndex entityIndex, final IndexIdentifier indexIdentifier ){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a22ef38..055af2a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
@@ -40,13 +39,7 @@ import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
-import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
-import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -60,8 +53,6 @@ import org.elasticsearch.index.query.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,7 +83,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private final EsProvider esProvider;
- private final IndexFig config;
//number of times to wait for the index to refresh properly.
private static final int MAX_WAITS = 10;
@@ -107,7 +97,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
private final IndexIdentifier indexIdentifier;
- private EsIndexCache aliasCache;
+ private IndexCache aliasCache;
private Timer mappingTimer;
private Timer refreshTimer;
private Meter refreshIndexMeter;
@@ -117,16 +107,15 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
@Inject
- public EsEntityIndexImpl( final IndexFig config,
+ public EsEntityIndexImpl(
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final IndexFig indexFig, final IndexIdentifier indexIdentifier ) {
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
this.esProvider = provider;
- this.config = config;
this.alias = indexIdentifier.getAlias();
this.aliasCache = indexCache;
this.addTimer = metricsFactory
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
index 5f5cc8b..2958d5c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
@@ -50,7 +50,7 @@ import com.google.inject.Singleton;
* Cache for Es index operations
*/
@Singleton
-public class EsIndexCacheImpl implements EsIndexCache {
+public class EsIndexCacheImpl implements IndexCache {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final ListeningScheduledExecutorService refreshExecutors;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index fe0cfa1..ebb79dc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -23,22 +23,16 @@ import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.EsIndexCache;
+import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
/**
* Classy class class.
*/
@@ -48,12 +42,12 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
private final EsProvider provider;
private final IndexFig indexFig;
private final IndexIdentifier indexIdentifier;
- private final EsIndexCache indexCache;
+ private final IndexCache indexCache;
private final VersionedData dataVersion;
private static final Logger log = LoggerFactory.getLogger(EsIndexDataMigrationImpl.class);
@Inject
- public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
+ public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, IndexCache indexCache){
this.entityIndex = entityIndex;
this.provider = provider;
this.indexFig = indexFig;
@@ -65,7 +59,9 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
final AdminClient adminClient = provider.getClient().admin();
+ final int latestVersion = dataVersion.getImplementationVersion();
+ observer.start();
migrationDataProvider.getData().flatMap(applicationScope -> {
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
@@ -76,11 +72,16 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ observer.update(latestVersion,"EsIndexDataMigrationImpl: fixed index: " + index );
+ })
+ .doOnError(error -> {
+ log.error("failed to migrate index", error);
+ observer.failed(latestVersion,"EsIndexDataMigrationImpl: failed to migrate",error);
})
- .doOnError(error -> log.error("failed to migrate index", error))
+ .doOnCompleted(() -> observer.complete())
.toBlocking().lastOrDefault(null);
- return dataVersion.getImplementationVersion();
+ return latestVersion;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 7a99d59..7b26376 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,7 +19,6 @@
package org.apache.usergrid.persistence.index.guice;
-import com.amazonaws.services.opsworks.model.App;
import com.google.inject.Inject;
import com.google.inject.TypeLiteral;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
@@ -29,11 +28,9 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.EsIndexCache;
+import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
-import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
@@ -63,17 +60,18 @@ public class TestIndexModule extends TestModule {
});
install( new GuicyFigModule(IndexTestFig.class) );
}
+
public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
@Inject
- public TestAllApplicationsObservable(final IndexFig config,
+ public TestAllApplicationsObservable(
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexCache indexCache, final MetricsFactory metricsFactory,
final IndexFig indexFig){
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,appScope);
- EntityIndex entityIndex = new EsEntityIndexImpl(config,indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
+ EntityIndex entityIndex = new EsEntityIndexImpl(indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
entityIndex.addIndex(null, 1, 0, indexFig.getWriteConsistencyLevel());
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b98cdce0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
new file mode 100644
index 0000000..47d81ed
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
@@ -0,0 +1,112 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
+import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.util.UUID;
+
+import static junit.framework.Assert.fail;
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Classy class class.
+ */
+@RunWith(EsRunner.class)
+@UseModules({ TestIndexModule.class })
+public class IndexMigrationTest extends BaseIT{
+
+
+ @Inject
+ public IndexFig fig;
+ @Inject
+ public IndexIdentifier indexIdentifier;
+
+ @Inject
+ public IndexCache indexCache;
+
+ @Inject
+ public EsProvider provider;
+
+ @Inject
+ public MigrationDataProvider<ApplicationScope> applicationScopeMigrationDataProvider;
+
+ @Inject
+ public AliasedEntityIndex ei;
+ @Inject
+ public IndexBufferProducer indexBatchBufferProducer;
+ @Inject
+ public MetricsFactory metricsFactory;
+
+ @Test
+ public void TestMigrate(){
+ EsIndexDataMigrationImpl indexDataMigration = new EsIndexDataMigrationImpl(ei,provider, fig, indexIdentifier,indexCache);
+ ProgressObserver po = new ProgressObserver() {
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void complete() {
+
+ }
+
+ @Override
+ public void failed(int migrationVersion, String reason) {
+ fail(reason);
+ }
+
+ @Override
+ public void failed(int migrationVersion, String reason, Throwable throwable) {
+ fail(reason);
+ }
+
+ @Override
+ public void update(int migrationVersion, String message) {
+
+ }
+ };
+ final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(fig,appScope);
+
+ TestIndexModule.TestAllApplicationsObservable obs = new TestIndexModule.TestAllApplicationsObservable(indexBatchBufferProducer,provider,indexCache,metricsFactory,fig);
+ int version = indexDataMigration.migrate(0, applicationScopeMigrationDataProvider, po );
+ assertEquals(version, IndexDataVersions.SINGLE_INDEX.getVersion());
+ }
+}
+
+
+
[15/24] incubator-usergrid git commit: adding test
Posted by to...@apache.org.
adding test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4b7b1220
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4b7b1220
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4b7b1220
Branch: refs/heads/two-dot-o-dev
Commit: 4b7b12203c62c6f6985d5fa21a7e042a24dfc353
Parents: 7976e1d
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:02:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:02:13 2015 -0600
----------------------------------------------------------------------
.../persistence/index/IndexIdentifier.java | 41 ++--------
.../persistence/index/IndexIdentifierImpl.java | 65 ++++++++++++++++
.../persistence/index/guice/IndexModule.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 -
.../migration/EsIndexDataMigrationImpl.java | 52 -------------
.../index/migration/LegacyIndexIdentifier.java | 78 ++++++++++++++++++++
.../index/guice/TestIndexModule.java | 26 ++++++-
7 files changed, 174 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 48c48f3..22b82de 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -17,46 +17,15 @@
* * directory of this distribution.
*
*/
-
package org.apache.usergrid.persistence.index;
-import com.google.inject.Inject;
-
/**
- * Class is used to generate an index name and alias name
+ * Classy class class.
*/
-public class IndexIdentifier{
- private final IndexFig config;
-
- @Inject
- public IndexIdentifier(IndexFig config) {
- this.config = config;
- }
-
- /**
- * Get the alias name
- * @return
- */
- public IndexAlias getAlias() {
- return new IndexAlias(config,config.getIndexPrefix());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- public String getIndex(String suffix) {
- if (suffix != null) {
- return config.getIndexPrefix() + "_" + suffix;
- } else {
- return config.getIndexPrefix();
- }
- }
-
+public interface IndexIdentifier {
+ IndexAlias getAlias();
- public String toString() {
- return "index id"+config.getIndexPrefix();
- }
+ String getIndex(String suffix);
+ String toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
new file mode 100644
index 0000000..1d18733
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
@@ -0,0 +1,65 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index;
+
+import com.google.inject.Inject;
+
+/**
+ * Class is used to generate an index name and alias name
+ */
+public class IndexIdentifierImpl implements IndexIdentifier {
+ private final IndexFig config;
+
+ @Inject
+ public IndexIdentifierImpl(IndexFig config) {
+ this.config = config;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ @Override
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,config.getIndexPrefix());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ @Override
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return config.getIndexPrefix() + "_" + suffix;
+ } else {
+ return config.getIndexPrefix();
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return "index id"+config.getIndexPrefix();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 7ecce54..aba49f6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -56,7 +56,7 @@ public abstract class IndexModule extends AbstractModule {
bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
- bind(IndexIdentifier.class);
+ bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 844260f..a22ef38 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -141,8 +141,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
}
-
-
@Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 13e6526..d6e3812 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -64,7 +64,6 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
- final AtomicInteger integer = new AtomicInteger();
final AdminClient adminClient = provider.getClient().admin();
migrationDataProvider.getData().flatMap(applicationScope -> {
@@ -77,7 +76,6 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
- integer.incrementAndGet();
})
.doOnError(error -> log.error("failed to migrate index", error))
.toBlocking().lastOrDefault(null);
@@ -94,55 +92,5 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
public int getMaxVersion() {
return dataVersion.getImplementationVersion();
}
- /**
- * Class is used to generate an index name and alias name the old way via app name
- */
- public class LegacyIndexIdentifier{
- private final IndexFig config;
- private final ApplicationScope applicationScope;
- public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
- this.config = config;
- this.applicationScope = applicationScope;
- }
-
- /**
- * Get the alias name
- * @return
- */
- public IndexAlias getAlias() {
- return new IndexAlias(config,getIndexBase());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- public String getIndex(String suffix) {
- if (suffix != null) {
- return getIndexBase() + "_" + suffix;
- } else {
- return getIndexBase();
- }
- }
-
- /**
- * returns the base name for index which will be used to add an alias and index
- * @return
- */
- private String getIndexBase() {
- StringBuilder sb = new StringBuilder();
- sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
- IndexingUtils.idString(sb, applicationScope.getApplication());
- return sb.toString();
- }
-
-
-
- public String toString() {
- return "application: " + applicationScope.getApplication().getUuid();
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
new file mode 100644
index 0000000..5ab3ac1
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/LegacyIndexIdentifier.java
@@ -0,0 +1,78 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+
+/**
+ * Class is used to generate an index name and alias name the old way via app name
+ */
+public class LegacyIndexIdentifier implements IndexIdentifier {
+ private final IndexFig config;
+ private final ApplicationScope applicationScope;
+
+ public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
+ this.config = config;
+ this.applicationScope = applicationScope;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,getIndexBase());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return getIndexBase() + "_" + suffix;
+ } else {
+ return getIndexBase();
+ }
+ }
+
+ /**
+ * returns the base name for index which will be used to add an alias and index
+ * @return
+ */
+ private String getIndexBase() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ IndexingUtils.idString(sb, applicationScope.getApplication());
+ return sb.toString();
+ }
+
+
+
+ public String toString() {
+ return "application: " + applicationScope.getApplication().getUuid();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b7b1220/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 0c57b36..6488b16 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,10 +19,22 @@
package org.apache.usergrid.persistence.index.guice;
+import com.amazonaws.services.opsworks.model.App;
+import com.google.inject.Inject;
import com.google.inject.TypeLiteral;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
+import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
@@ -53,11 +65,23 @@ public class TestIndexModule extends TestModule {
}
public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
+ final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
+
+ @Inject
+ public TestAllApplicationsObservable(final IndexFig config,
+ final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
+ final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final IndexFig indexFig){
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,appScope);
+ EntityIndex entityIndex = new EsEntityIndexImpl(config,indexBatchBufferProducer,provider,indexCache,metricsFactory,indexFig,legacyIndexIdentifier);
+ entityIndex.addIndex(null, 1, 0, indexFig.getWriteConsistencyLevel());
+ }
+
@Override
public Observable<ApplicationScope> getData() {
ApplicationScope[] scopes = new ApplicationScope[]{
- new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"))
+ appScope
};
return Observable.from(scopes);
}
[05/24] incubator-usergrid git commit: change query to support legacy
context
Posted by to...@apache.org.
change query to support legacy context
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/90832ca0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/90832ca0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/90832ca0
Branch: refs/heads/two-dot-o-dev
Commit: 90832ca0ccab39c6d170d7d42eae3314468f5d17
Parents: e0a79db
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 16:18:56 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 16:18:56 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EntityToMapConverter.java | 8 ++++++--
.../index/impl/EsApplicationEntityIndexImpl.java | 3 ++-
.../usergrid/persistence/index/impl/IndexRequest.java | 4 +---
.../usergrid/persistence/index/impl/IndexingUtils.java | 8 +++++++-
.../apache/usergrid/persistence/index/query/Query.java | 13 +++++++++----
.../persistence/query/tree/GrammarTreeTest.java | 4 ++--
6 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 6e72159..fdd9b92 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -16,6 +16,7 @@
*/
package org.apache.usergrid.persistence.index.impl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
@@ -35,7 +36,7 @@ public class EntityToMapConverter {
* @param entity The entity
* @param context The context this entity appears in
*/
- public static Map convert( final Entity entity, final String context ) {
+ public static Map convert(ApplicationScope applicationScope, final Entity entity, final String context ) {
final Map entityMap = entityToMap( entity );
//add the context for filtering later
@@ -43,7 +44,10 @@ public class EntityToMapConverter {
//but the fieldname we have to prefix because we use query equality to seek this later.
// TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
+ entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
+
+ entityMap.put( APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()) );
+
return entityMap;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 1debe3b..ade2286 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -145,10 +145,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
+ String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
.setTypes(searchTypes.getTypeNames(applicationScope))
.setScroll(cursorTimeout + "m")
- .setQuery(query.createQueryBuilder(createContextName(applicationScope, indexScope)));
+ .setQuery(query.createQueryBuilder(contexts));
final FilterBuilder fb = query.createFilterBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 9eae481..dc05820 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -55,16 +55,14 @@ public class IndexRequest implements BatchRequest {
}
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, String context , Entity entity) {
- this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(entity, context));
+ this(writeAlias, applicationScope, SearchType.fromId(entity.getId()),IndexingUtils.createIndexDocId(entity,context), EntityToMapConverter.convert(applicationScope,entity, context));
}
public IndexRequest( final String writeAlias, final ApplicationScope applicationScope,SearchType searchType, String documentId, Map<String, Object> data) {
- data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
this.entityType = searchType.getTypeName(applicationScope);
this.data = data;
this.documentId = documentId;
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 2fb56d1..0330bbf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -76,7 +76,13 @@ public class IndexingUtils {
sb.append( scope.getName() );
return sb.toString();
}
-
+ public static String createLegacyContextName(ApplicationScope applicationScope, IndexScope scope ) {
+ StringBuilder sb = new StringBuilder();
+ idString(sb, scope.getOwner());
+ sb.append( SEPARATOR );
+ sb.append( scope.getName() );
+ return sb.toString();
+ }
/**
* Append the id to the string
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 67a1731..438d82e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -60,6 +60,7 @@ import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@@ -147,7 +148,7 @@ public class Query {
}
- public QueryBuilder createQueryBuilder( final String context ) {
+ public QueryBuilder createQueryBuilder( final String[] contexts ) {
QueryBuilder queryBuilder = null;
@@ -178,14 +179,18 @@ public class Query {
// TODO evaluate performance when it's an all query.
// Do we need to put the context term first for performance?
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for(String context : contexts){
+ boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
+ }
+ boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
if ( queryBuilder != null ) {
- queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
- .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+ queryBuilder = boolQueryBuilder.must( queryBuilder );
}
//nothing was specified ensure we specify the context in the search
else {
- queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ queryBuilder = boolQueryBuilder;
}
return queryBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/90832ca0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 34185b3..102e222 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -411,7 +411,7 @@ public class GrammarTreeTest {
@Test
public void selectGeoWithAnd() throws RecognitionException {
String queryString = "select * where location within 20000 of 37,-75 "
- + "and created > 1407776999925 and created < 1407777000266";
+ + "and created > 1407776999925 and created < 1407777000266";
ANTLRStringStream in = new ANTLRStringStream( queryString );
CpQueryFilterLexer lexer = new CpQueryFilterLexer( in );
@@ -429,7 +429,7 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- QueryBuilder qb = query.createQueryBuilder("testcontext");
+ QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"});
}
[13/24] incubator-usergrid git commit: fix cursor
Posted by to...@apache.org.
fix cursor
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2df40137
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2df40137
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2df40137
Branch: refs/heads/two-dot-o-dev
Commit: 2df40137c9d96ec05f82605e77ebcaeba016a420
Parents: 96d27b9
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 14:23:04 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 14:23:04 2015 -0600
----------------------------------------------------------------------
.../impl/EsApplicationEntityIndexImpl.java | 11 +++++++---
.../migration/EsIndexDataMigrationImpl.java | 2 +-
.../index/guice/TestIndexModule.java | 22 ++++++++++++++++++++
3 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 603bee4..53286b4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -142,6 +142,10 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
@Override
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit){
+ if(query.getCursor()!=null){
+ return getNextPage(query.getCursor());
+ }
+
SearchResponse searchResponse;
SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query,limit);
@@ -291,11 +295,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
candidates.add(new CandidateResult(entityId, UUID.fromString(version)));
}
- CandidateResults candidateResults = new CandidateResults(candidates);
- if(candidateResults.hasCursor()) {
+ final CandidateResults candidateResults = new CandidateResults(candidates);
+ final String esScrollCursor = searchResponse.getScrollId();
+
+ if(esScrollCursor != null) {
candidateResults.initializeCursor();
- final String esScrollCursor = searchResponse.getScrollId();
//now set this into our map module
final int minutes = indexFig.getQueryCursorTimeout();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 2a0bb15..64b7d29 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -80,7 +80,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
.doOnError(error -> log.error("failed to migrate index",error))
.toBlocking().last();
- return integer.get();
+ return dataVersion.getImplementationVersion();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2df40137/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 4e6700c..0c57b36 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,10 +19,18 @@
package org.apache.usergrid.persistence.index.guice;
+import com.google.inject.TypeLiteral;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
+import rx.Observable;
+
+import java.util.UUID;
public class TestIndexModule extends TestModule {
@@ -37,8 +45,22 @@ public class TestIndexModule extends TestModule {
@Override
public void configureMigrationProvider(){
+ bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
+ TestAllApplicationsObservable.class );
}
});
install( new GuicyFigModule(IndexTestFig.class) );
}
+ public static class TestAllApplicationsObservable implements MigrationDataProvider<ApplicationScope>{
+
+
+ @Override
+ public Observable<ApplicationScope> getData() {
+ ApplicationScope[] scopes = new ApplicationScope[]{
+ new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"))
+ };
+ return Observable.from(scopes);
+ }
+ }
+
}
[12/24] incubator-usergrid git commit: move back to old Query object
Posted by to...@apache.org.
move back to old Query object
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96d27b95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96d27b95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96d27b95
Branch: refs/heads/two-dot-o-dev
Commit: 96d27b95336b41aee16180fe80fbf9bc9fb21f4d
Parents: c09e6b4
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 13:34:01 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 13:34:01 2015 -0600
----------------------------------------------------------------------
.../index/ApplicationEntityIndex.java | 6 +-
.../usergrid/persistence/index/EntityIndex.java | 2 +
.../impl/EsApplicationEntityIndexImpl.java | 9 +-
.../impl/SearchRequestBuilderStrategy.java | 11 +-
.../usergrid/persistence/index/query/Query.java | 184 ++++++++++++++++---
5 files changed, 174 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 2167efa..59a19eb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -43,11 +43,9 @@ public interface ApplicationEntityIndex {
* Execute query in Usergrid syntax.
*/
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query);
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit);
+
- /**
- * Execute query in Usergrid syntax.
- */
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit );
/**
* get next page of results
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index af6b013..6783fb0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -39,6 +39,8 @@ import java.util.concurrent.Future;
public interface EntityIndex extends CPManager {
+ public static final int MAX_LIMIT = 1000;
+
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 27791ae..603bee4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -136,16 +136,15 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){
- return search(indexScope,searchTypes,query,10);
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query) {
+ return search(indexScope,searchTypes,query,query.getLimit());
}
-
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit){
SearchResponse searchResponse;
- SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit);
+ SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query,limit);
if (logger.isDebugEnabled()) {
logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 15796f9..207a7a8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -22,10 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.IndexAlias;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
@@ -56,7 +53,6 @@ public class SearchRequestBuilderStrategy {
private final ApplicationScope applicationScope;
private final IndexAlias alias;
private final int cursorTimeout;
- public static final int MAX_LIMIT = 1000;
public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
@@ -66,8 +62,9 @@ public class SearchRequestBuilderStrategy {
this.cursorTimeout = cursorTimeout;
}
- public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
- Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT);
+ public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+
+ Preconditions.checkArgument(limit <= EntityIndex.MAX_LIMIT, "limit is greater than max "+ EntityIndex.MAX_LIMIT);
SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
.setTypes(searchTypes.getTypeNames(applicationScope))
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96d27b95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index fa7def4..5567382 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -40,7 +40,10 @@ import org.antlr.runtime.Token;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -52,10 +55,14 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual;
import org.apache.usergrid.persistence.index.query.tree.LessThan;
import org.apache.usergrid.persistence.index.query.tree.LessThanEqual;
import org.apache.usergrid.persistence.index.query.tree.Operand;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +79,9 @@ public class Query {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
+ public static final int DEFAULT_LIMIT = 10;
+ public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
@@ -80,6 +89,8 @@ public class Query {
private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>();
private Operand rootOperand;
private UUID startResult;
+ private String cursor;
+ private int limit = 0;
private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
private boolean mergeSelectResults = false;
@@ -111,19 +122,22 @@ public class Query {
* @param q
*/
public Query( Query q ) {
- if (q == null) {
+ if ( q == null ) {
return;
}
+
type = q.type;
sortPredicates = q.sortPredicates != null
- ? new ArrayList<SortPredicate>(q.sortPredicates) : null;
+ ? new ArrayList<>( q.sortPredicates ) : null;
startResult = q.startResult;
+ cursor = q.cursor;
+ limit = q.limit;
selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<String, String>(q.selectAssignments) : null;
+ ? new LinkedHashMap<>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
//level = q.level;
connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<String>(q.permissions) : null;
+ permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
reversed = q.reversed;
reversedSet = q.reversedSet;
startTime = q.startTime;
@@ -132,16 +146,76 @@ public class Query {
pad = q.pad;
rootOperand = q.rootOperand;
identifiers = q.identifiers != null
- ? new ArrayList<Identifier>(q.identifiers) : null;
+ ? new ArrayList<>( q.identifiers ) : null;
counterFilters = q.counterFilters != null
- ? new ArrayList<CounterFilterPredicate>(q.counterFilters) : null;
+ ? new ArrayList<>( q.counterFilters ) : null;
collection = q.collection;
-
level = q.level;
}
+ public QueryBuilder createQueryBuilder( final String context ) {
+
+
+ QueryBuilder queryBuilder = null;
+
+
+ //we have a root operand. Translate our AST into an ES search
+ if ( getRootOperand() != null ) {
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
+ // will return a tile query + post filter
+ QueryVisitor v = new EsQueryVistor();
+
+ try {
+ getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+
+
+ queryBuilder = v.getQueryBuilder();
+ }
+
+
+ // Add our filter for context to our query for fast execution.
+ // Fast because it utilizes bitsets internally. See this post for more detail.
+ // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+
+ // TODO evaluate performance when it's an all query.
+ // Do we need to put the context term first for performance?
+ if ( queryBuilder != null ) {
+ queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
+ .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+ }
+
+ //nothing was specified ensure we specify the context in the search
+ else {
+ queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+ }
+
+ return queryBuilder;
+ }
+
+
+ public FilterBuilder createFilterBuilder() {
+ FilterBuilder filterBuilder = null;
+
+ if ( getRootOperand() != null ) {
+ QueryVisitor v = new EsQueryVistor();
+ try {
+ getRootOperand().visit( v );
+
+ } catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+ filterBuilder = v.getFilterBuilder();
+ }
+
+ return filterBuilder;
+ }
/**
@@ -169,8 +243,8 @@ public class Query {
String qlt = ql.toLowerCase();
if ( !qlt.startsWith( "select" )
- && !qlt.startsWith( "insert" )
- && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) {
+ && !qlt.startsWith( "insert" )
+ && !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) {
if ( qlt.startsWith( "order by" ) ) {
ql = "select * " + ql;
@@ -224,7 +298,7 @@ public class Query {
if ( o instanceof Map ) {
@SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, List<String>> params =
- ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
+ ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
return fromQueryParams( params );
}
return null;
@@ -232,7 +306,7 @@ public class Query {
public static Query fromQueryParams( Map<String, List<String>> params )
- throws QueryParseException {
+ throws QueryParseException {
Query q = null;
CounterResolution resolution = null;
List<Identifier> identifiers = null;
@@ -243,6 +317,8 @@ public class Query {
Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
+ String cursor = ListUtils.first( params.get( "cursor" ) );
+ Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
@@ -313,7 +389,15 @@ public class Query {
q.setStartResult( start );
}
+ if ( cursor != null ) {
+ q = newQueryIfNull( q );
+ q.setCursor( cursor );
+ }
+ if ( limit != null ) {
+ q = newQueryIfNull( q );
+ q.setLimit( limit );
+ }
if ( startTime != null ) {
q = newQueryIfNull( q );
@@ -356,21 +440,22 @@ public class Query {
public static Query searchForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter(propertyName, propertyValue);
+ q.addEqualityFilter( propertyName, propertyValue );
return q;
}
public static Query findForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter(propertyName, propertyValue);
+ q.addEqualityFilter( propertyName, propertyValue );
+ q.setLimit( 1 );
return q;
}
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID(uuid) );
+ q.addIdentifier( Identifier.fromUUID( uuid ) );
return q;
}
@@ -629,8 +714,8 @@ public class Query {
for ( SortPredicate s : sortPredicates ) {
if ( s.getPropertyName().equals( propertyName ) ) {
logger.error(
- "Attempted to set sort order for " + s.getPropertyName()
- + " more than once, discarding..." );
+ "Attempted to set sort order for " + s.getPropertyName()
+ + " more than once, discarding..." );
return this;
}
}
@@ -727,8 +812,8 @@ public class Query {
public Query addContainsFilter( String propName, String keyword ) {
ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) );
- equality.setProperty(propName);
- equality.setLiteral(keyword);
+ equality.setProperty( propName );
+ equality.setLiteral( keyword );
addClause( equality );
@@ -737,8 +822,8 @@ public class Query {
private void addClause( EqualityOperand equals, String propertyName, Object value ) {
- equals.setProperty(propertyName);
- equals.setLiteral(value);
+ equals.setProperty( propertyName );
+ equals.setLiteral( value );
addClause( equals );
}
@@ -800,7 +885,7 @@ public class Query {
}
- public UUID getStartResult(String cursor) {
+ public UUID getStartResult() {
if ( ( startResult == null ) && ( cursor != null ) ) {
byte[] cursorBytes = Base64.decodeBase64( cursor );
if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
@@ -811,6 +896,61 @@ public class Query {
}
+ public String getCursor() {
+ return cursor;
+ }
+
+
+ public void setCursor( String cursor ) {
+ this.cursor = cursor;
+ }
+
+
+ public Query withCursor( String cursor ) {
+ setCursor( cursor );
+ return this;
+ }
+
+
+ public int getLimit() {
+ return getLimit( DEFAULT_LIMIT );
+ }
+
+
+ public int getLimit( int defaultLimit ) {
+ if ( limit <= 0 ) {
+ if ( defaultLimit > 0 ) {
+ return defaultLimit;
+ }
+ else {
+ return DEFAULT_LIMIT;
+ }
+ }
+ return limit;
+ }
+
+
+ public void setLimit( int limit ) {
+
+ // TODO tnine. After users have had time to change their query limits,
+ // this needs to be uncommented and enforced.
+ // if(limit > MAX_LIMIT){
+ // throw new IllegalArgumentException(
+ // String.format("Query limit must be <= to %d", MAX_LIMIT));
+ // }
+
+ if ( limit > MAX_LIMIT ) {
+ limit = MAX_LIMIT;
+ }
+
+ this.limit = limit;
+ }
+
+
+ public Query withLimit( int limit ) {
+ setLimit( limit );
+ return this;
+ }
public boolean isReversed() {
@@ -998,7 +1138,7 @@ public class Query {
public SortPredicate(@JsonProperty("propertyName") String propertyName,
- @JsonProperty("direction") Query.SortDirection direction ) {
+ @JsonProperty("direction") Query.SortDirection direction ) {
if ( propertyName == null ) {
throw new NullPointerException( "Property name was null" );
[20/24] incubator-usergrid git commit: change all application
observable
Posted by to...@apache.org.
change all application observable
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/32cb153d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/32cb153d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/32cb153d
Branch: refs/heads/two-dot-o-dev
Commit: 32cb153d1712d112c9ca0eafb775b42b92413c9d
Parents: 6a39add
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 09:41:02 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 09:41:02 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/AllApplicationsObservable.java | 5 -----
.../corepersistence/rx/impl/AbstractGraphVisitorImpl.java | 2 +-
.../rx/impl/AllApplicationsObservableImpl.java | 7 ++-----
.../corepersistence/rx/ApplicationObservableTestIT.java | 2 +-
.../src/main/java/org/apache/usergrid/rest/RootResource.java | 2 +-
5 files changed, 5 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
index 055223e..24f32f3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
@@ -32,9 +32,4 @@ import rx.Observable;
*/
public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
- /**
- * Return all applications in our system
- * @return
- */
- public Observable<ApplicationScope> getAllApplications();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
index b8cb80b..ef66481 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AbstractGraphVisitorImpl.java
@@ -58,7 +58,7 @@ public abstract class AbstractGraphVisitorImpl<T> implements MigrationDataProvid
@Override
public Observable<T> getData() {
- return applicationObservable.getAllApplications().flatMap( new Func1<ApplicationScope, Observable<T>>() {
+ return applicationObservable.getData().flatMap( new Func1<ApplicationScope, Observable<T>>() {
@Override
public Observable<T> call( final ApplicationScope applicationScope ) {
return getAllEntities( applicationScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index aa5b5dc..b57f088 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -69,7 +69,7 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
@Override
- public Observable<ApplicationScope> getAllApplications() {
+ public Observable<ApplicationScope> getData() {
//emit our 3 hard coded applications that are used the manage the system first.
//this way consumers can perform whatever work they need to on the root system first
@@ -138,8 +138,5 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
}
- @Override
- public Observable<ApplicationScope> getData() {
- return getAllApplications();
- }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
index 649f518..9572079 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/ApplicationObservableTestIT.java
@@ -67,7 +67,7 @@ public class ApplicationObservableTestIT extends AbstractCoreIT {
//clean up our wiring
ManagerCache managerCache = SpringResource.getInstance().getBean( Injector.class ).getInstance( ManagerCache.class );
- Observable<ApplicationScope> appObservable = applicationObservable.getAllApplications();
+ Observable<ApplicationScope> appObservable = applicationObservable.getData();
appObservable.doOnNext( new Action1<ApplicationScope>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32cb153d/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
index e4086f9..3fa4f74 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/RootResource.java
@@ -114,7 +114,7 @@ public class RootResource extends AbstractContextResource implements MetricProce
@QueryParam("deleted") @DefaultValue("false") Boolean deleted,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws URISyntaxException {
- logger.info( "RootResource.getAllApplications" );
+ logger.info( "RootResource.getData" );
ApiResponse response = createApiResponse();
response.setAction( "get applications" );
[02/24] incubator-usergrid git commit: remove array prefix
Posted by to...@apache.org.
remove array prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1a61e0ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1a61e0ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1a61e0ee
Branch: refs/heads/two-dot-o-dev
Commit: 1a61e0eec82da613226396ab8cd533fd5353a89c
Parents: cc779b2
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 23 11:16:41 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 23 11:16:41 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a61e0ee/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 3b65c74..3a6f019 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -245,12 +245,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put(ARRAY_PREFIX + field.getName().toLowerCase(),
+ entityMap.put(field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
@@ -262,7 +262,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put(SET_PREFIX+ field.getName().toLowerCase(),
+ entityMap.put( field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
[23/24] incubator-usergrid git commit: add comments for interfaces
Posted by to...@apache.org.
add comments for interfaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3a318366
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3a318366
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3a318366
Branch: refs/heads/two-dot-o-dev
Commit: 3a31836627b55aef920e32c8e125d31d70a9dedf
Parents: c18ba59
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:14:35 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:14:35 2015 -0600
----------------------------------------------------------------------
.../core/metrics/MetricsFactory.java | 2 +-
.../index/ApplicationEntityIndex.java | 2 +-
.../persistence/index/IndexBufferConsumer.java | 2 +-
.../persistence/index/IndexBufferProducer.java | 2 +-
.../usergrid/persistence/index/SearchType.java | 2 +-
.../migration/EsIndexDataMigrationImpl.java | 40 +++++++++++---------
6 files changed, 28 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
index 62a5cb9..b79f33d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactory.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.persistence.core.metrics;
import com.codahale.metrics.*;
/**
- * Classy class class.
+ * Get metrics .
*/
public interface MetricsFactory {
MetricRegistry getRegistry();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 59a19eb..cb1b6f0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.persistence.index.query.Query;
import rx.Observable;
/**
- * Classy class class.
+ * Entity Index for an Application.
*/
public interface ApplicationEntityIndex {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
index 40c7852..e0fb8b3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferConsumer.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Buffer index requests
*/
public interface IndexBufferConsumer {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
index 7c8b7e6..a9e6d6f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
@@ -28,7 +28,7 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
- * Classy class class.
+ * Produce to index buffer consumer
*/
public interface IndexBufferProducer {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
index c039360..786219a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
@@ -24,7 +24,7 @@ import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.model.entity.Id;
/**
- * Classy class class.
+ * Get search type
*/
public class SearchType{
private final String type;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a318366/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index ebb79dc..123527a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.index.IndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.indices.InvalidAliasNameException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -62,24 +63,29 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
final int latestVersion = dataVersion.getImplementationVersion();
observer.start();
- migrationDataProvider.getData().flatMap(applicationScope -> {
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
- String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
- return Observable.from(indexes);
- })
- .doOnNext(index -> {
- IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
- aliasesRequestBuilder = adminClient.indices().prepareAliases();
- // add read alias
- aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
- observer.update(latestVersion,"EsIndexDataMigrationImpl: fixed index: " + index );
+ try {
+ migrationDataProvider.getData().flatMap(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ return Observable.from(indexes);
})
- .doOnError(error -> {
- log.error("failed to migrate index", error);
- observer.failed(latestVersion,"EsIndexDataMigrationImpl: failed to migrate",error);
- })
- .doOnCompleted(() -> observer.complete())
- .toBlocking().lastOrDefault(null);
+ .doOnNext(index -> {
+ IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ // add read alias
+ try {
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ } catch (InvalidAliasNameException e) {
+ log.debug("Failed to add alias due to name conflict",e);
+ }
+ observer.update(latestVersion, "EsIndexDataMigrationImpl: fixed index: " + index);
+ })
+ .doOnCompleted(() -> observer.complete())
+ .toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ log.error("Failed to migrate index", e);
+ throw e;
+ }
return latestVersion;
}
[16/24] incubator-usergrid git commit: move impl
Posted by to...@apache.org.
move impl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9e8d506c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9e8d506c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9e8d506c
Branch: refs/heads/two-dot-o-dev
Commit: 9e8d506c3bc5d96a6f24a32f01a3fa7b54531a51
Parents: 4b7b122
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:10:03 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:10:03 2015 -0600
----------------------------------------------------------------------
.../persistence/index/IndexIdentifierImpl.java | 65 -------------------
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/IndexIdentifierImpl.java | 68 ++++++++++++++++++++
3 files changed, 69 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
deleted file mode 100644
index 1d18733..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifierImpl.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.index;
-
-import com.google.inject.Inject;
-
-/**
- * Class is used to generate an index name and alias name
- */
-public class IndexIdentifierImpl implements IndexIdentifier {
- private final IndexFig config;
-
- @Inject
- public IndexIdentifierImpl(IndexFig config) {
- this.config = config;
- }
-
- /**
- * Get the alias name
- * @return
- */
- @Override
- public IndexAlias getAlias() {
- return new IndexAlias(config,config.getIndexPrefix());
- }
-
- /**
- * Get index name, send in additional parameter to add incremental indexes
- * @param suffix
- * @return
- */
- @Override
- public String getIndex(String suffix) {
- if (suffix != null) {
- return config.getIndexPrefix() + "_" + suffix;
- } else {
- return config.getIndexPrefix();
- }
- }
-
-
- @Override
- public String toString() {
- return "index id"+config.getIndexPrefix();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index aba49f6..922a916 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -26,13 +26,8 @@ import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
-import com.google.inject.assistedinject.FactoryModuleBuilder;
-import org.apache.usergrid.persistence.index.impl.BufferQueue;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
-import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.index.impl.*;
import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin;
import org.apache.usergrid.persistence.index.migration.IndexMigration;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e8d506c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
new file mode 100644
index 0000000..0139858
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexIdentifierImpl.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+
+/**
+ * Class is used to generate an index name and alias name
+ */
+public class IndexIdentifierImpl implements IndexIdentifier {
+ private final IndexFig config;
+
+ @Inject
+ public IndexIdentifierImpl(IndexFig config) {
+ this.config = config;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ @Override
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,config.getIndexPrefix());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ @Override
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return config.getIndexPrefix() + "_" + suffix;
+ } else {
+ return config.getIndexPrefix();
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return "index id"+config.getIndexPrefix();
+ }
+
+}
[03/24] incubator-usergrid git commit: remove context
Posted by to...@apache.org.
remove context
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a9eafb87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a9eafb87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a9eafb87
Branch: refs/heads/two-dot-o-dev
Commit: a9eafb8757988cb875f4677eaeeb261615ccc95f
Parents: 1a61e0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 11:16:36 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 11:16:36 2015 -0600
----------------------------------------------------------------------
.../data/DataMigrationManagerImpl.java | 9 +-
.../persistence/index/impl/DeIndexRequest.java | 42 ++--
.../index/impl/EntityToMapConverter.java | 169 ++++++++++++++
.../impl/EsApplicationEntityIndexImpl.java | 9 +-
.../index/impl/EsEntityIndexBatchImpl.java | 220 +------------------
.../persistence/index/impl/IndexRequest.java | 19 +-
6 files changed, 221 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
index ad87724..158315d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/DataMigrationManagerImpl.java
@@ -98,13 +98,10 @@ public class DataMigrationManagerImpl implements DataMigrationManager {
/**
* Invoke each plugin to attempt a migration
*/
- for(final MigrationPlugin plugin: executionOrder){
+ executionOrder.forEach(plugin -> {
final ProgressObserver observer = new CassandraProgressObserver(plugin.getName());
-
- plugin.run( observer );
- }
-
-
+ plugin.run(observer);
+ });
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
index 9f3ce66..98aa8cf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexRequest.java
@@ -21,7 +21,13 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Arrays;
+import java.util.UUID;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.client.Client;
@@ -30,6 +36,9 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+
/**
* Represent the properties required to build an index request
@@ -37,19 +46,21 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
@JsonTypeInfo(use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class")
public class DeIndexRequest implements BatchRequest {
- public String[] indexes;
- public String entityType;
- public String documentId;
+ private String[] entityTypes;
+ private String[] indexes;
+ private String documentId;
- public DeIndexRequest( final String[] indexes, final String entityType, final String documentId) {
- this.indexes = indexes;
- this.entityType = entityType;
- this.documentId = documentId;
+ public DeIndexRequest( ) {
+
}
- public DeIndexRequest() {
+ public DeIndexRequest(String[] indexes, ApplicationScope applicationScope, IndexScope indexScope, Id id, UUID version) {
+ String context = createContextName(applicationScope,indexScope);
+ this.indexes = indexes;
+ this.entityTypes = SearchType.fromId(id).getTypeNames(applicationScope);
+ this.documentId = createIndexDocId(id, version,context);
}
@@ -58,9 +69,10 @@ public class DeIndexRequest implements BatchRequest {
for(final String index: indexes) {
- final DeleteRequestBuilder builder = client.prepareDelete( index, entityType, documentId);
-
- bulkRequest.add( builder );
+ for(String entityType : entityTypes) {
+ final DeleteRequestBuilder builder = client.prepareDelete(index, entityType, documentId);
+ bulkRequest.add(builder);
+ }
}
}
@@ -70,8 +82,8 @@ public class DeIndexRequest implements BatchRequest {
}
- public String getEntityType() {
- return entityType;
+ public String[] getEntityTypes() {
+ return entityTypes;
}
@@ -94,7 +106,7 @@ public class DeIndexRequest implements BatchRequest {
if ( !documentId.equals( that.documentId ) ) {
return false;
}
- if ( !entityType.equals( that.entityType ) ) {
+ if ( !entityTypes.equals( that.entityTypes ) ) {
return false;
}
if ( !Arrays.equals( indexes, that.indexes ) ) {
@@ -108,7 +120,7 @@ public class DeIndexRequest implements BatchRequest {
@Override
public int hashCode() {
int result = Arrays.hashCode( indexes );
- result = 31 * result + entityType.hashCode();
+ result = 31 * result + entityTypes.hashCode();
result = 31 * result + documentId.hashCode();
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
new file mode 100644
index 0000000..6e72159
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.util.*;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+
+/**
+ * Classy class class.
+ */
+public class EntityToMapConverter {
+ /**
+ * Set the entity as a map with the context
+ *
+ * @param entity The entity
+ * @param context The context this entity appears in
+ */
+ public static Map convert( final Entity entity, final String context ) {
+ final Map entityMap = entityToMap( entity );
+
+ //add the context for filtering later
+ entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
+
+ //but the fieldname we have to prefix because we use query equality to seek this later.
+ // TODO see if we can make this more declarative
+ entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
+
+ return entityMap;
+ }
+
+
+ /**
+ * Convert Entity to Map and Adding prefixes for types:
+ * <pre>
+ * su_ - String unanalyzed field
+ * sa_ - String analyzed field
+ * go_ - Location field nu_ - Number field
+ * bu_ - Boolean field
+ * </pre>
+ */
+ private static Map entityToMap( EntityObject entity ) {
+
+ Map<String, Object> entityMap = new HashMap<String, Object>();
+
+ for ( Object f : entity.getFields().toArray() ) {
+
+ Field field = ( Field ) f;
+
+
+ if ( f instanceof ArrayField) {
+ List list = ( List ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+ }
+ else if ( f instanceof ListField) {
+ List list = ( List ) field.getValue();
+ entityMap.put(field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+
+ if ( !list.isEmpty() ) {
+ if ( list.get( 0 ) instanceof String ) {
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
+ }
+ }
+ }
+ else if ( f instanceof SetField) {
+ Set set = ( Set ) field.getValue();
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( set ) ) );
+ }
+ else if ( f instanceof EntityObjectField) {
+ EntityObject eo = ( EntityObject ) field.getValue();
+ entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
+ }
+ else if ( f instanceof StringField ) {
+
+ // index in lower case because Usergrid queries are case insensitive
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ ( ( String ) field.getValue() ).toLowerCase() );
+ }
+ else if ( f instanceof LocationField ) {
+ LocationField locField = ( LocationField ) f;
+ Map<String, Object> locMap = new HashMap<String, Object>();
+
+ // field names lat and lon trigger ElasticSearch geo location
+ locMap.put( "lat", locField.getValue().getLatitude() );
+ locMap.put( "lon", locField.getValue().getLongitude() );
+ entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
+ }
+ else if( f instanceof DoubleField || f instanceof FloatField){
+ entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if( f instanceof LongField || f instanceof IntegerField){
+ entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof BooleanField ) {
+
+ entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
+ }
+ else if ( f instanceof UUIDField ) {
+
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ field.getValue().toString().toLowerCase() );
+ }
+ else {
+ entityMap.put( field.getName().toLowerCase(), field.getValue() );
+ }
+ }
+
+ return entityMap;
+ }
+
+
+ private static Collection processCollectionForMap( final Collection c ) {
+ if ( c.isEmpty() ) {
+ return c;
+ }
+ List processed = new ArrayList();
+ Object sample = c.iterator().next();
+
+ if ( sample instanceof Entity ) {
+ for ( Object o : c.toArray() ) {
+ Entity e = ( Entity ) o;
+ processed.add( entityToMap( e ) );
+ }
+ }
+ else if ( sample instanceof List ) {
+ for ( Object o : c.toArray() ) {
+ List list = ( List ) o;
+ processed.add( processCollectionForMap( list ) ); // recursion;
+ }
+ }
+ else if ( sample instanceof Set ) {
+ for ( Object o : c.toArray() ) {
+ Set set = ( Set ) o;
+ processed.add( processCollectionForMap( set ) ); // recursion;
+ }
+ }
+ else {
+ for ( Object o : c.toArray() ) {
+ processed.add( o );
+ }
+ }
+ return processed;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 2d01697..4c3e42c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -148,16 +148,13 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName(applicationScope,indexScope);
- final String[] entityTypes = searchTypes.getTypeNames(applicationScope);
- QueryBuilder qb = query.createQueryBuilder(context);
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(entityTypes)
+ .setTypes(searchTypes.getTypeNames(applicationScope))
.setScroll(cursorTimeout + "m")
- .setQuery(qb);
+ .setQuery(query.createQueryBuilder(createContextName(applicationScope, indexScope)));
final FilterBuilder fb = query.createFilterBuilder();
@@ -216,7 +213,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
- this.alias.getReadAlias(), context, entityTypes, srb );
+ this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb );
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 3a6f019..8f8ac5e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -21,11 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.*;
import org.apache.usergrid.persistence.core.future.BetterFuture;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.*;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,28 +31,7 @@ import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.BooleanField;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.FloatField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.ListField;
-import org.apache.usergrid.persistence.model.field.LocationField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.SetField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
-import org.apache.usergrid.persistence.model.field.value.EntityObject;
-
-import com.codahale.metrics.*;
-import com.codahale.metrics.Timer;
-
-import rx.Observable;
-import rx.functions.Func1;
-
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -95,33 +70,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyEntityWrite( entity );
ValidationUtils.verifyVersion( entity.getVersion() );
//add app id for indexing
-
- final String context = createContextName(applicationScope,indexScope);
-
- if ( log.isDebugEnabled() ) {
- log.debug( "Indexing entity {}:{}\n alias: {}\n" +
- " app: {}\n scope owner: {}\n scope name: {}\n context: {}",
- entity.getId().getType(), entity.getId().getUuid(), alias.getWriteAlias(),
- applicationScope.getApplication(), indexScope.getOwner(), indexScope.getName(), context );
- }
-
- ValidationUtils.verifyEntityWrite( entity );
-
- Map<String, Object> entityAsMap = entityToMap( entity, context );
- //add app id
- entityAsMap.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
- // need prefix here because we index UUIDs as strings
-
- // let caller add these fields if needed
- // entityAsMap.put("created", entity.getId().getUuid().timestamp();
- // entityAsMap.put("updated", entity.getVersion().timestamp());
-
- String indexId = createIndexDocId( entity, context );
-
- log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- final SearchType entityType =SearchType.fromId(entity.getId());
- container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType.getTypeName(applicationScope), indexId, entityAsMap));
-
+ container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), applicationScope,indexScope, entity));
return this;
}
@@ -133,38 +82,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyIdentity(id);
ValidationUtils.verifyVersion( version );
- final String context = createContextName(applicationScope,indexScope);
- final SearchType entityType =SearchType.fromId(id);
-
- final String indexId = createIndexDocId( id, version, context );
-
-
- if ( log.isDebugEnabled() ) {
- log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n "
- + "name {} context{}, type {},",
- new Object[] {
- id.getType(),
- id.getUuid(),
- applicationScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(),
- context,
- entityType
- } );
- }
-
- String[] indexes = entityIndex.getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] indexes = entityIndex.getUniqueIndexes();
//get the default index if no alias exists yet
if(indexes == null ||indexes.length == 0){
indexes = new String[]{indexIdentifier.getIndex(null)};
}
-
- String[] typeNames = entityType.getTypeNames(applicationScope);
- for(String type : typeNames) {
- container.addDeIndexRequest(new DeIndexRequest(indexes, type, indexId));
- }
- log.debug("Deindexed Entity with index id " + indexId);
+ container.addDeIndexRequest(new DeIndexRequest(indexes, applicationScope,indexScope,id,version));
return this;
}
@@ -205,142 +129,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- /**
- * Set the entity as a map with the context
- *
- * @param entity The entity
- * @param context The context this entity appears in
- */
- private static Map entityToMap( final Entity entity, final String context ) {
- final Map entityMap = entityToMap( entity );
-
- //add the context for filtering later
- entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
-
- //but the fieldname we have to prefix because we use query equality to seek this later.
- // TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase());
-
- return entityMap;
- }
- /**
- * Convert Entity to Map and Adding prefixes for types:
- * <pre>
- * su_ - String unanalyzed field
- * sa_ - String analyzed field
- * go_ - Location field nu_ - Number field
- * bu_ - Boolean field
- * </pre>
- */
- private static Map entityToMap( EntityObject entity ) {
-
- Map<String, Object> entityMap = new HashMap<String, Object>();
-
- for ( Object f : entity.getFields().toArray() ) {
-
- Field field = ( Field ) f;
-
-
- if ( f instanceof ArrayField ) {
- List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
- }
- else if ( f instanceof ListField ) {
- List list = ( List ) field.getValue();
- entityMap.put(field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
-
- if ( !list.isEmpty() ) {
- if ( list.get( 0 ) instanceof String ) {
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
- }
- }
- }
- else if ( f instanceof SetField ) {
- Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( set ) ) );
- }
- else if ( f instanceof EntityObjectField ) {
- EntityObject eo = ( EntityObject ) field.getValue();
- entityMap.put(EO_PREFIX + field.getName().toLowerCase(), entityToMap(eo) ); // recursion
- }
- else if ( f instanceof StringField ) {
-
- // index in lower case because Usergrid queries are case insensitive
- entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- ( ( String ) field.getValue() ).toLowerCase() );
- }
- else if ( f instanceof LocationField ) {
- LocationField locField = ( LocationField ) f;
- Map<String, Object> locMap = new HashMap<String, Object>();
-
- // field names lat and lon trigger ElasticSearch geo location
- locMap.put( "lat", locField.getValue().getLatitude() );
- locMap.put( "lon", locField.getValue().getLongitude() );
- entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
- }
- else if( f instanceof DoubleField || f instanceof FloatField){
- entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if( f instanceof LongField || f instanceof IntegerField){
- entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof BooleanField ) {
-
- entityMap.put( BOOLEAN_PREFIX + field.getName().toLowerCase(), field.getValue() );
- }
- else if ( f instanceof UUIDField ) {
-
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
- field.getValue().toString().toLowerCase() );
- }
- else {
- entityMap.put( field.getName().toLowerCase(), field.getValue() );
- }
- }
-
- return entityMap;
- }
-
-
- private static Collection processCollectionForMap( final Collection c ) {
- if ( c.isEmpty() ) {
- return c;
- }
- List processed = new ArrayList();
- Object sample = c.iterator().next();
-
- if ( sample instanceof Entity ) {
- for ( Object o : c.toArray() ) {
- Entity e = ( Entity ) o;
- processed.add( entityToMap( e ) );
- }
- }
- else if ( sample instanceof List ) {
- for ( Object o : c.toArray() ) {
- List list = ( List ) o;
- processed.add( processCollectionForMap( list ) ); // recursion;
- }
- }
- else if ( sample instanceof Set ) {
- for ( Object o : c.toArray() ) {
- Set set = ( Set ) o;
- processed.add( processCollectionForMap( set ) ); // recursion;
- }
- }
- else {
- for ( Object o : c.toArray() ) {
- processed.add( o );
- }
- }
- return processed;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a9eafb87/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
index 4ec4092..23f3d08 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexRequest.java
@@ -22,12 +22,20 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Map;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.idString;
+
/**
* Represent the properties required to build an index request
@@ -42,11 +50,14 @@ public class IndexRequest implements BatchRequest {
public Map<String, Object> data;
- public IndexRequest( final String writeAlias, final String entityType, final String documentId,
- final Map<String, Object> data ) {
+ public IndexRequest( final String writeAlias, final ApplicationScope applicationScope, IndexScope indexScope, Entity entity) {
+ String context = createContextName(applicationScope,indexScope);
+ SearchType searchType = SearchType.fromId(entity.getId());
+ final Map<String, Object> data = EntityToMapConverter.convert(entity,context);
+ data.put(APPLICATION_ID_FIELDNAME, idString(applicationScope.getApplication()));
this.writeAlias = writeAlias;
- this.entityType = entityType;
- this.documentId = documentId;
+ this.entityType = searchType.getTypeName(applicationScope);
+ this.documentId = IndexingUtils.createIndexDocId(entity,context);
this.data = data;
}
[22/24] incubator-usergrid git commit: add comments for interfaces
Posted by to...@apache.org.
add comments for interfaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c18ba594
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c18ba594
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c18ba594
Branch: refs/heads/two-dot-o-dev
Commit: c18ba594dc6e88662a30d525b2ddee34d1ed30d7
Parents: 33b1b69
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:08:28 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:08:28 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexCache.java | 12 +++++++++++-
.../usergrid/persistence/index/IndexIdentifier.java | 16 +++++++++++++++-
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c18ba594/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
index 253c66d..e852230 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexCache.java
@@ -20,10 +20,20 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Cache for index lookups
*/
public interface IndexCache {
+ /**
+ * get index based on alias name
+ * @param alias
+ * @param aliasType
+ * @return
+ */
String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+ /**
+ * invalidate cache
+ * @param alias
+ */
void invalidate(IndexAlias alias);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c18ba594/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index 22b82de..f393ee3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -20,12 +20,26 @@
package org.apache.usergrid.persistence.index;
/**
- * Classy class class.
+ * Identifier for where an index is in underlying server
*/
public interface IndexIdentifier {
+
+ /**
+ * get the alias name
+ * @return
+ */
IndexAlias getAlias();
+ /**
+ * get index name from suffix
+ * @param suffix
+ * @return
+ */
String getIndex(String suffix);
+ /**
+ * return unique string
+ * @return
+ */
String toString();
}
[19/24] incubator-usergrid git commit: fix indexcacheimpl
Posted by to...@apache.org.
fix indexcacheimpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6a39add9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6a39add9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6a39add9
Branch: refs/heads/two-dot-o-dev
Commit: 6a39add920a052bf629c86df3d30615f59e0c22d
Parents: b98cdce
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 16:04:00 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 16:04:00 2015 -0600
----------------------------------------------------------------------
.../serialization/impl/migration/EdgeDataMigrationImpl.java | 2 ++
.../usergrid/persistence/index/impl/IndexMigrationTest.java | 8 +-------
2 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a39add9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index ecb9a9b..411e318 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -88,6 +88,8 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
+ //TODO: maybe move to retreive all indexes : https://github.com/elastic/elasticsearch/blob/master/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+ //https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=get%20all%20indexes%20elasticsearch
//get edges from the source
return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6a39add9/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
index 47d81ed..1203e38 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexMigrationTest.java
@@ -60,9 +60,6 @@ public class IndexMigrationTest extends BaseIT{
public EsProvider provider;
@Inject
- public MigrationDataProvider<ApplicationScope> applicationScopeMigrationDataProvider;
-
- @Inject
public AliasedEntityIndex ei;
@Inject
public IndexBufferProducer indexBatchBufferProducer;
@@ -98,12 +95,9 @@ public class IndexMigrationTest extends BaseIT{
}
};
- final ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId(UUID.randomUUID(),"application"));
-
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(fig,appScope);
TestIndexModule.TestAllApplicationsObservable obs = new TestIndexModule.TestAllApplicationsObservable(indexBatchBufferProducer,provider,indexCache,metricsFactory,fig);
- int version = indexDataMigration.migrate(0, applicationScopeMigrationDataProvider, po );
+ int version = indexDataMigration.migrate(0, obs, po );
assertEquals(version, IndexDataVersions.SINGLE_INDEX.getVersion());
}
}
[08/24] incubator-usergrid git commit: add data migration
Posted by to...@apache.org.
add data migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f695724b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f695724b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f695724b
Branch: refs/heads/two-dot-o-dev
Commit: f695724b0417131eb9592a642ae8fa7a29afd8cd
Parents: 8a62d11
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:14:12 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:14:12 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexAlias.java | 38 ++++++
.../persistence/index/IndexIdentifier.java | 18 ---
.../persistence/index/guice/IndexModule.java | 19 +++
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EsIndexCache.java | 7 +-
.../impl/SearchRequestBuilderStrategy.java | 5 +-
.../migration/EsIndexDataMigrationImpl.java | 134 +++++++++++++++++++
.../index/migration/EsIndexMigrationPlugin.java | 46 +++++++
.../index/migration/IndexMigration.java | 35 +++++
10 files changed, 281 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
new file mode 100644
index 0000000..a04f80e
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Abstraction for Index alias names
+ */
+public class IndexAlias{
+ private final String readAlias;
+ private final String writeAlias;
+
+ public IndexAlias(IndexFig indexFig,String indexBase) {
+ this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix();
+ this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix();
+ }
+
+ public String getReadAlias() {
+ return readAlias;
+ }
+
+ public String getWriteAlias() {
+ return writeAlias;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index c659ed6..48c48f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -55,24 +55,6 @@ public class IndexIdentifier{
}
- public class IndexAlias{
- private final String readAlias;
- private final String writeAlias;
-
- public IndexAlias(IndexFig indexFig,String indexBase) {
- this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix();
- this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix();
- }
-
- public String getReadAlias() {
- return readAlias;
- }
-
- public String getWriteAlias() {
- return writeAlias;
- }
- }
-
public String toString() {
return "index id"+config.getIndexPrefix();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index c9125c5..a42dea8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -19,6 +19,11 @@
package org.apache.usergrid.persistence.index.guice;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -28,6 +33,9 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
+import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin;
+import org.apache.usergrid.persistence.index.migration.IndexMigration;
import org.apache.usergrid.persistence.map.guice.MapModule;
import org.apache.usergrid.persistence.queue.guice.QueueModule;
@@ -56,6 +64,17 @@ public class IndexModule extends AbstractModule {
bind( BufferQueue.class).toProvider( QueueProvider.class );
+
+ //wire up the edg migration
+ Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class );
+
+
+ dataMigrationMultibinder.addBinding().to(EsIndexDataMigrationImpl.class);
+
+
+ //wire up the collection migration plugin
+ Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index ca071e2..27791ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -76,7 +76,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final EsIndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final Timer deleteApplicationTimer;
private final Meter deleteApplicationMeter;
private final SearchRequestBuilderStrategy searchRequest;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 259fa55..cab8ded 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -77,7 +77,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public static final String DEFAULT_TYPE = "_default_";
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final IndexBufferProducer indexBatchBufferProducer;
private final IndexFig indexFig;
private final Timer addTimer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index ef518dd..9ba89ca 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -27,6 +27,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.apache.usergrid.persistence.index.IndexAlias;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.cluster.metadata.AliasMetaData;
@@ -98,7 +99,7 @@ public class EsIndexCache {
/**
* Get indexes for an alias
*/
- public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ public String[] getIndexes( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
String[] indexes;
try {
indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
@@ -127,7 +128,7 @@ public class EsIndexCache {
* @param aliasType
* @return
*/
- private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
}
@@ -135,7 +136,7 @@ public class EsIndexCache {
/**
* clean up cache
*/
- public void invalidate( IndexIdentifier.IndexAlias alias ) {
+ public void invalidate( IndexAlias alias ) {
aliasIndexCache.invalidate( alias.getWriteAlias() );
aliasIndexCache.invalidate( alias.getReadAlias() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index d146e4c..15796f9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchTypes;
@@ -53,11 +54,11 @@ public class SearchRequestBuilderStrategy {
private final EsProvider esProvider;
private final ApplicationScope applicationScope;
- private final IndexIdentifier.IndexAlias alias;
+ private final IndexAlias alias;
private final int cursorTimeout;
public static final int MAX_LIMIT = 1000;
- public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){
+ public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
this.esProvider = esProvider;
this.applicationScope = applicationScope;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
new file mode 100644
index 0000000..b5dab53
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexAlias;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
+import org.elasticsearch.client.AdminClient;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Classy class class.
+ */
+public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope> {
+
+ private final AliasedEntityIndex entityIndex;
+ private final EsProvider provider;
+ private final IndexFig indexFig;
+ private final IndexIdentifier indexIdentifier;
+ private final EsIndexCache indexCache;
+
+ @Inject
+ public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
+ this.entityIndex = entityIndex;
+ this.provider = provider;
+ this.indexFig = indexFig;
+ this.indexIdentifier = indexIdentifier;
+ this.indexCache = indexCache;
+ }
+
+ @Override
+ public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
+ migrationDataProvider.getData().doOnNext(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ AdminClient adminClient = provider.getClient().admin();
+
+ for (String index : indexes) {
+ IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ aliasesRequestBuilder = adminClient.indices().prepareAliases();
+ // add read alias
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ }
+ });
+ return 0;
+ }
+
+ @Override
+ public boolean supports(int currentVersion) {
+ return false;
+ }
+
+ @Override
+ public int getMaxVersion() {
+ return 0;
+ }
+ /**
+ * Class is used to generate an index name and alias name the old way via app name
+ */
+ public class LegacyIndexIdentifier{
+ private final IndexFig config;
+ private final ApplicationScope applicationScope;
+
+ public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) {
+ this.config = config;
+ this.applicationScope = applicationScope;
+ }
+
+ /**
+ * Get the alias name
+ * @return
+ */
+ public IndexAlias getAlias() {
+ return new IndexAlias(config,getIndexBase());
+ }
+
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @param suffix
+ * @return
+ */
+ public String getIndex(String suffix) {
+ if (suffix != null) {
+ return getIndexBase() + "_" + suffix;
+ } else {
+ return getIndexBase();
+ }
+ }
+
+ /**
+ * returns the base name for index which will be used to add an alias and index
+ * @return
+ */
+ private String getIndexBase() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
+ IndexingUtils.idString(sb, applicationScope.getApplication());
+ return sb.toString();
+ }
+
+
+
+ public String toString() {
+ return "application: " + applicationScope.getApplication().getUuid();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
new file mode 100644
index 0000000..a28c701
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.core.migration.data.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.Set;
+
+/**
+ * Classy class class.
+ */
+public class EsIndexMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope>{
+
+ @Inject
+ public EsIndexMigrationPlugin(@IndexMigration final Set<DataMigration<ApplicationScope>> entityDataMigrations,
+ final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ){
+ super(entityDataMigrations,entityIdScopeDataMigrationProvider,migrationInfoSerialization);
+ }
+
+ @Override
+ public String getName() {
+ return "index-migration";
+ }
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
new file mode 100644
index 0000000..c398aac
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence.index.migration;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Index migration annotation
+ */
+
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface IndexMigration {}
[11/24] incubator-usergrid git commit: add version migration for
indexes
Posted by to...@apache.org.
add version migration for indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c09e6b4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c09e6b4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c09e6b4d
Branch: refs/heads/two-dot-o-dev
Commit: c09e6b4d8d3213c54976184593e6bb592b0c130b
Parents: bf0718e
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 11:27:34 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 11:27:34 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/index/migration/IndexDataVersions.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c09e6b4d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
index 921d86a..6558339 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
@@ -21,7 +21,7 @@
package org.apache.usergrid.persistence.index.migration;
/**
- * Classy class class.
+ * Index data migration
*/
public enum IndexDataVersions {
MANY_INDEXES(0),
[17/24] incubator-usergrid git commit: fix indexcacheimpl
Posted by to...@apache.org.
fix indexcacheimpl
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a7e08db8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a7e08db8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a7e08db8
Branch: refs/heads/two-dot-o-dev
Commit: a7e08db85babdb1f4c75c38cee67ba6f03547e32
Parents: 9e8d506
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 15:17:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 15:17:13 2015 -0600
----------------------------------------------------------------------
.../persistence/index/EsIndexCache.java | 29 ++++
.../persistence/index/guice/IndexModule.java | 4 +-
.../persistence/index/impl/EsIndexCache.java | 143 -------------------
.../index/impl/EsIndexCacheImpl.java | 141 ++++++++++++++++++
.../migration/EsIndexDataMigrationImpl.java | 2 +-
.../index/guice/TestIndexModule.java | 2 +-
6 files changed, 175 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
new file mode 100644
index 0000000..6427459
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EsIndexCache.java
@@ -0,0 +1,29 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index;
+
+/**
+ * Classy class class.
+ */
+public interface EsIndexCache {
+ String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType);
+
+ void invalidate(IndexAlias alias);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 922a916..bb5ec54 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -48,9 +48,11 @@ public abstract class IndexModule extends AbstractModule {
install(new MapModule());
install(new QueueModule());
- bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
+ bind(EntityIndexFactory.class).to(EsEntityIndexFactoryImpl.class);
bind(AliasedEntityIndex.class).to(EsEntityIndexImpl.class);
bind(EntityIndex.class).to(EsEntityIndexImpl.class);
+ bind(EsIndexCache.class).to(EsIndexCacheImpl.class);
+
bind(IndexIdentifier.class).to(IndexIdentifierImpl.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
deleted file mode 100644
index 9ba89ca..0000000
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.usergrid.persistence.index.IndexAlias;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListenableFutureTask;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- * Cache for Es index operations
- */
-@Singleton
-public class EsIndexCache {
-
- private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
- private final ListeningScheduledExecutorService refreshExecutors;
-
- private LoadingCache<String, String[]> aliasIndexCache;
- private EsProvider provider;
-
-
- @Inject
- public EsIndexCache( final EsProvider provider, final IndexFig indexFig ) {
-
- this.refreshExecutors =
- MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
-
- this.provider = provider;
-
- aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
- .build( new CacheLoader<String, String[]>() {
- @Override
- public ListenableFuture<String[]> reload( final String key,
- String[] oldValue )
- throws Exception {
- ListenableFutureTask<String[]> task =
- ListenableFutureTask.create( new Callable<String[]>() {
- public String[] call() {
- return load( key );
- }
- } );
- refreshExecutors.execute( task );
- return task;
- }
-
-
- @Override
- public String[] load( final String aliasName ) {
- return getIndexesFromEs(aliasName);
- }
- } );
- }
-
-
- /**
- * Get indexes for an alias
- */
- public String[] getIndexes( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
- String[] indexes;
- try {
- indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
- }
- catch ( ExecutionException ee ) {
- logger.error( "Failed to retreive indexes", ee );
- throw new RuntimeException( ee );
- }
- return indexes;
- }
-
-
-
- private String[] getIndexesFromEs(final String aliasName){
- final AdminClient adminClient = this.provider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
- adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
- return aliasMap.keys().toArray( String.class );
- }
-
-
- /**
- * Get the name of the alias to use
- * @param alias
- * @param aliasType
- * @return
- */
- private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
- return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
- }
-
-
- /**
- * clean up cache
- */
- public void invalidate( IndexAlias alias ) {
- aliasIndexCache.invalidate( alias.getWriteAlias() );
- aliasIndexCache.invalidate( alias.getReadAlias() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
new file mode 100644
index 0000000..5f5cc8b
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCacheImpl.java
@@ -0,0 +1,141 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.usergrid.persistence.index.*;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ListeningScheduledExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * Cache for Es index operations
+ */
+@Singleton
+public class EsIndexCacheImpl implements EsIndexCache {
+
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
+ private final ListeningScheduledExecutorService refreshExecutors;
+
+ private LoadingCache<String, String[]> aliasIndexCache;
+ private EsProvider provider;
+
+
+ @Inject
+ public EsIndexCacheImpl( final EsProvider provider, final IndexFig indexFig ) {
+
+ this.refreshExecutors =
+ MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
+
+ this.provider = provider;
+
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
+ .build( new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload( final String key,
+ String[] oldValue )
+ throws Exception {
+ ListenableFutureTask<String[]> task =
+ ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute( task );
+ return task;
+ }
+
+
+ @Override
+ public String[] load( final String aliasName ) {
+ return getIndexesFromEs(aliasName);
+ }
+ } );
+ }
+
+
+ /**
+ * Get indexes for an alias
+ */
+ @Override
+ public String[] getIndexes(IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ String[] indexes;
+ try {
+ indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
+ }
+ catch ( ExecutionException ee ) {
+ logger.error( "Failed to retreive indexes", ee );
+ throw new RuntimeException( ee );
+ }
+ return indexes;
+ }
+
+
+
+ private String[] getIndexesFromEs(final String aliasName){
+ final AdminClient adminClient = this.provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
+ adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
+ return aliasMap.keys().toArray( String.class );
+ }
+
+
+ /**
+ * Get the name of the alias to use
+ * @param alias
+ * @param aliasType
+ * @return
+ */
+ private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
+ }
+
+
+ /**
+ * clean up cache
+ */
+ @Override
+ public void invalidate(IndexAlias alias) {
+ aliasIndexCache.invalidate( alias.getWriteAlias() );
+ aliasIndexCache.invalidate( alias.getReadAlias() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index d6e3812..fe0cfa1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -26,7 +26,7 @@ import org.apache.usergrid.persistence.index.AliasedEntityIndex;
import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.EsIndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a7e08db8/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 6488b16..7a99d59 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -31,7 +31,7 @@ import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexIdentifier;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
-import org.apache.usergrid.persistence.index.impl.EsIndexCache;
+import org.apache.usergrid.persistence.index.EsIndexCache;
import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl;
import org.apache.usergrid.persistence.index.migration.LegacyIndexIdentifier;
[06/24] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-501
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-501
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e5eb57fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e5eb57fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e5eb57fd
Branch: refs/heads/two-dot-o-dev
Commit: e5eb57fd80525def86211e7a99cdc71be93a8e5a
Parents: 90832ca 66fdc61
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 24 16:37:55 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 24 16:37:55 2015 -0600
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 26 +-
.../applications/ApplicationResource.java | 44 +-
.../applications/imports/ImportsResource.java | 4 -
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +--------
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 4 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +++++-------
.../usergrid/rest/management/AdminUsersIT.java | 60 +-
.../rest/management/ExportResourceIT.java | 652 +++++++------------
.../rest/management/ImportResourceIT.java | 20 +-
.../rest/management/OrganizationsIT.java | 16 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/NamedResource.java | 112 +++-
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 18 +-
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/MeResource.java | 32 +
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
30 files changed, 809 insertions(+), 1115 deletions(-)
----------------------------------------------------------------------
[07/24] incubator-usergrid git commit: change entityindex interface
Posted by to...@apache.org.
change entityindex interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a62d117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a62d117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a62d117
Branch: refs/heads/two-dot-o-dev
Commit: 8a62d1177783612189be49fb924dfead3f348e2f
Parents: e5eb57f
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 25 14:26:10 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 25 14:26:10 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 122 ++++++------
.../index/ApplicationEntityIndex.java | 13 +-
.../impl/EsApplicationEntityIndexImpl.java | 199 +++++++------------
.../impl/SearchRequestBuilderStrategy.java | 197 ++++++++++++++++++
.../index/query/CandidateResults.java | 24 ++-
.../usergrid/persistence/index/query/Query.java | 160 +--------------
.../persistence/index/impl/EntityIndexTest.java | 28 +--
.../persistence/query/tree/GrammarTreeTest.java | 8 +-
8 files changed, 382 insertions(+), 369 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f420d7b..481fff5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -753,14 +753,14 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Application app ) throws Exception {
- update( app );
+ update(app);
this.application = app;
}
@Override
public void updateApplication( Map<String, Object> properties ) throws Exception {
- this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties );
+ this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties);
this.application = get( applicationId, Application.class );
}
@@ -857,7 +857,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
- return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
+ return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias);
}
@@ -886,7 +886,7 @@ public class CpEntityManager implements EntityManager {
new Object[] { ownerRef, collectionType, aliasValue } );
}
- return results.get( aliasValue );
+ return results.get(aliasValue);
}
@@ -936,7 +936,7 @@ public class CpEntityManager implements EntityManager {
}
- return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid()));
}
@@ -1003,7 +1003,7 @@ public class CpEntityManager implements EntityManager {
public void setProperty(
EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
- setProperty( entityRef, propertyName, propertyValue, false );
+ setProperty(entityRef, propertyName, propertyValue, false);
}
@@ -1021,7 +1021,7 @@ public class CpEntityManager implements EntityManager {
entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
- entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
+ entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID()));
update( entity );
}
@@ -1127,7 +1127,7 @@ public class CpEntityManager implements EntityManager {
public void addToDictionary( EntityRef entityRef, String dictionaryName,
Object elementValue ) throws Exception {
- addToDictionary( entityRef, dictionaryName, elementValue, null );
+ addToDictionary(entityRef, dictionaryName, elementValue, null);
}
@@ -1188,7 +1188,7 @@ public class CpEntityManager implements EntityManager {
EntityRef entity = get( entityRef );
UUID timestampUuid = UUIDUtils.newTimeUUID();
- Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
+ Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be);
for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(),
@@ -1225,8 +1225,8 @@ public class CpEntityManager implements EntityManager {
boolean coTypeIsBasic = ClassUtils.isBasicType( setCoType );
List<HColumn<ByteBuffer, ByteBuffer>> results =
- cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be );
+ cass.getAllColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), be, be);
for ( HColumn<ByteBuffer, ByteBuffer> result : results ) {
Object name = null;
if ( entityHasDictionary ) {
@@ -1275,7 +1275,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF dictionaryCf = null;
- boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName );
+ boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName);
if ( entityHasDictionary ) {
dictionaryCf = ENTITY_DICTIONARIES;
@@ -1285,14 +1285,14 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
HColumn<ByteBuffer, ByteBuffer> result =
- cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
- entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ),
- be, be );
+ cass.getColumn(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName),
+ entityHasDictionary ? bytebuffer(elementName) : DynamicComposite.toByteBuffer(elementName),
+ be, be);
if ( result != null ) {
if ( entityHasDictionary && coTypeIsBasic ) {
@@ -1303,7 +1303,7 @@ public class CpEntityManager implements EntityManager {
}
}
else {
- logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" );
+ logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null");
}
return value;
@@ -1327,8 +1327,8 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
- boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType);
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
for ( int i = 0; i < elementNames.length; i++ ) {
@@ -1337,8 +1337,8 @@ public class CpEntityManager implements EntityManager {
}
ColumnSlice<ByteBuffer, ByteBuffer> results =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+ cass.getColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), columnNames, be, be);
if ( results != null ) {
values = new HashMap<String, Object>();
for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1384,7 +1384,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getDictionaries( EntityRef entity ) throws Exception {
- return getDictionaryNames( entity );
+ return getDictionaryNames(entity);
}
@@ -1398,14 +1398,14 @@ public class CpEntityManager implements EntityManager {
@Override
public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isCollectionMember( collectionName, entity );
+ return getRelationManager( owner ).isCollectionMember(collectionName, entity);
}
@Override
public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isConnectionMember( connectionName, entity );
+ return getRelationManager( owner ).isConnectionMember(connectionName, entity);
}
@@ -1430,7 +1430,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( get( entityId ))
- .getCollection ( collectionName, query, resultsLevel );
+ .getCollection(collectionName, query, resultsLevel);
}
@@ -1495,8 +1495,8 @@ public class CpEntityManager implements EntityManager {
@Override
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
- return createConnection( connection.getConnectingEntity(), connection.getConnectionType(),
- connection.getConnectedEntity() );
+ return createConnection(connection.getConnectingEntity(), connection.getConnectionType(),
+ connection.getConnectedEntity());
}
@@ -1504,7 +1504,7 @@ public class CpEntityManager implements EntityManager {
public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
- return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef );
+ return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef);
}
@@ -1514,7 +1514,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( connectingEntity )
- .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1540,7 +1540,7 @@ public class CpEntityManager implements EntityManager {
EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception {
return getRelationManager( connectingEntity )
- .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1572,7 +1572,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectedEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1581,7 +1581,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectingEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1589,21 +1589,21 @@ public class CpEntityManager implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String entityType, Level level, int count ) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
+ return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count);
}
@Override
public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities( query );
+ return getRelationManager( connectingEntity ).searchConnectedEntities(query);
}
@Override
public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception {
- return getRelationManager( entity ).getConnectionIndexes( connectionType );
+ return getRelationManager( entity ).getConnectionIndexes(connectionType);
}
@@ -1829,14 +1829,14 @@ public class CpEntityManager implements EntityManager {
String roleTitle = batchRoleName;
String propertyName = groupId + ":" + batchRoleName;
Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
- properties.put( "group", groupId );
+ properties.put("group", groupId);
- Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties );
+ Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties);
getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) )
- .addToCollection( COLLECTION_ROLES, entity );
+ .addToCollection(COLLECTION_ROLES, entity);
- logger.info( "Created role {} with id {} in group {}",
- new String[] { roleName, entity.getUuid().toString(), groupId.toString() } );
+ logger.info("Created role {} with id {} in group {}",
+ new String[]{roleName, entity.getUuid().toString(), groupId.toString()});
return entity;
}
@@ -1848,8 +1848,8 @@ public class CpEntityManager implements EntityManager {
permission = permission.toLowerCase();
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
+ CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp);
//Adding graphite metrics
Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
@@ -1927,7 +1927,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantUserPermission( UUID userId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission);
}
@@ -1951,9 +1951,9 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
EntityRef userRef = userRef( userId );
EntityRef roleRef = getRoleRef( roleName );
- addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( userRef, COLLECTION_ROLES, roleRef );
- addToCollection( roleRef, COLLECTION_USERS, userRef );
+ addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(userRef, COLLECTION_ROLES, roleRef);
+ addToCollection(roleRef, COLLECTION_USERS, userRef);
}
@@ -1988,8 +1988,8 @@ public class CpEntityManager implements EntityManager {
private EntityRef getRoleRef( String roleName ) throws Exception {
- Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) );
+ Results results = this.searchCollection(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ Schema.defaultCollectionName(Role.ENTITY_TYPE), Query.findForProperty("roleName", roleName));
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2235,8 +2235,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getCounterNames() throws Exception {
Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
- Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) );
- names.addAll( nameSet );
+ Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS));
+ names.addAll(nameSet);
return names;
}
@@ -2300,7 +2300,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
getApplicationScope().getApplication(), collectionName);
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collectionScope);
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
@@ -2313,7 +2313,7 @@ public class CpEntityManager implements EntityManager {
public Entity get( UUID uuid ) throws Exception {
MapManager mm = getMapManagerForTypes();
- String entityType = mm.getString( uuid.toString() );
+ String entityType = mm.getString(uuid.toString());
final Entity entity;
@@ -2335,9 +2335,9 @@ public class CpEntityManager implements EntityManager {
private MapManager getMapManagerForTypes() {
Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
- MapManager mm = managerCache.getMapManager( ms );
+ MapManager mm = managerCache.getMapManager(ms);
return mm;
}
@@ -2440,7 +2440,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) );
}
@@ -2455,8 +2455,8 @@ public class CpEntityManager implements EntityManager {
@Override
public void addGroupToRole( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) );
+ addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName));
}
@@ -2477,7 +2477,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantGroupPermission( UUID groupId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission);
}
@@ -2663,8 +2663,8 @@ public class CpEntityManager implements EntityManager {
catch ( HystrixRuntimeException hre ) {
if ( hre.getCause() instanceof WriteUniqueVerifyException ) {
- WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause();
- handleWriteUniqueVerifyException( entity, wuve );
+ WriteUniqueVerifyException wuve = (WriteUniqueVerifyException) hre.getCause();
+ handleWriteUniqueVerifyException(entity, wuve);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index dd1c5fa..2167efa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -39,11 +39,22 @@ public interface ApplicationEntityIndex {
*/
public EntityIndexBatch createBatch();
+ /**
+ * Execute query in Usergrid syntax.
+ */
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query);
/**
* Execute query in Usergrid syntax.
*/
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit );
+
+ /**
+ * get next page of results
+ * @param cursor
+ * @return
+ */
+ public CandidateResults getNextPage(final String cursor);
/**
* delete all application records
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index ade2286..ca071e2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -46,15 +46,9 @@ import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -85,6 +79,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final IndexIdentifier.IndexAlias alias;
private final Timer deleteApplicationTimer;
private final Meter deleteApplicationMeter;
+ private final SearchRequestBuilderStrategy searchRequest;
private FailureMonitor failureMonitor;
private final int cursorTimeout;
@Inject
@@ -117,6 +112,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
this.alias = indexIdentifier.getAlias();
+ this.searchRequest = new SearchRequestBuilderStrategy(esProvider,appScope,alias,cursorTimeout);
+
}
@Override
@@ -139,132 +136,78 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
- final Query query ) {
-
- SearchResponse searchResponse;
-
- if ( query.getCursor() == null ) {
- String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames(applicationScope))
- .setScroll(cursorTimeout + "m")
- .setQuery(query.createQueryBuilder(contexts));
-
- final FilterBuilder fb = query.createFilterBuilder();
-
- //we have post filters, apply them
- if ( fb != null ) {
- logger.debug( " Filter: {} ", fb.toString() );
- srb = srb.setPostFilter( fb );
- }
-
-
- srb = srb.setFrom( 0 ).setSize( query.getLimit() );
-
- for ( Query.SortPredicate sp : query.getSortPredicates() ) {
-
- final SortOrder order;
- if ( sp.getDirection().equals( Query.SortDirection.ASCENDING ) ) {
- order = SortOrder.ASC;
- }
- else {
- order = SortOrder.DESC;
- }
-
- // we do not know the type of the "order by" property and so we do not know what
- // type prefix to use. So, here we add an order by clause for every possible type
- // that you can order by: string, number and boolean and we ask ElasticSearch
- // to ignore any fields that are not present.
-
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
- .order( order ).ignoreUnmapped( true );
- srb.addSort( stringSort );
-
- logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){
+ return search(indexScope,searchTypes,query,10);
+ }
- final String longFieldName = LONG_PREFIX + sp.getPropertyName();
- final FieldSortBuilder longSort = SortBuilders.fieldSort( longFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( longSort );
- logger.debug( " Sort: {} order by {}", longFieldName, order.toString() );
+ @Override
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ SearchResponse searchResponse;
- final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
- final FieldSortBuilder doubleSort = SortBuilders.fieldSort( doubleFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( doubleSort );
- logger.debug( " Sort: {} order by {}", doubleFieldName, order.toString() );
+ SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
+ this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb);
+ }
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( booleanSort );
- logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
- }
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearch = searchTimer.time();
+ searchResponse = srb.execute().actionGet();
+ timeSearch.stop();
+ } catch (Throwable t) {
+ logger.error("Unable to communicate with Elasticsearch", t);
+ failureMonitor.fail("Unable to execute batch", t);
+ throw t;
+ }
+ failureMonitor.success();
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
- this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb );
- }
+ return parseResults(searchResponse);
+ }
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
- timeSearch.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ public CandidateResults getNextPage(final String cursor){
+ SearchResponse searchResponse;
- failureMonitor.success();
+ String userCursorString = cursor;
+ if ( userCursorString.startsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 1 );
+ }
+ if ( userCursorString.endsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
}
- else {
- String userCursorString = query.getCursor();
- if ( userCursorString.startsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 1 );
- }
- if ( userCursorString.endsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
- }
-
- //now get the cursor from the map and validate
- final String esScrollCursor = mapManager.getString( userCursorString );
-
- Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
+ //now get the cursor from the map and validate
+ final String esScrollCursor = mapManager.getString(userCursorString);
+ Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
- logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- SearchScrollRequestBuilder ssrb = esProvider.getClient()
- .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" );
+ logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet();
- timeSearchCursor.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ SearchScrollRequestBuilder ssrb = esProvider.getClient()
+ .prepareSearchScroll(esScrollCursor).setScroll(cursorTimeout + "m");
- failureMonitor.success();
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearchCursor = cursorTimer.time();
+ searchResponse = ssrb.execute().actionGet();
+ timeSearchCursor.stop();
+ }
+ catch ( Throwable t ) {
+ logger.error( "Unable to communicate with elasticsearch", t );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
}
- return parseResults(searchResponse, query);
+
+ failureMonitor.success();
+ return parseResults(searchResponse);
}
/**
@@ -327,7 +270,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
- private CandidateResults parseResults( final SearchResponse searchResponse, final Query query ) {
+ private CandidateResults parseResults( final SearchResponse searchResponse) {
final SearchHits searchHits = searchResponse.getHits();
final SearchHit[] hits = searchHits.getHits();
@@ -335,41 +278,39 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
logger.debug(" Hit count: {} Total hits: {}", length, searchHits.getTotalHits());
- List<CandidateResult> candidates = new ArrayList<>( length );
+ List<CandidateResult> candidates = new ArrayList<>(length);
- for ( SearchHit hit : hits ) {
+ for (SearchHit hit : hits) {
- String[] idparts = hit.getId().split( SPLITTER );
+ String[] idparts = hit.getId().split(SPLITTER);
String id = idparts[0];
String type = idparts[1];
String version = idparts[2];
- Id entityId = new SimpleId( UUID.fromString(id), type );
+ Id entityId = new SimpleId(UUID.fromString(id), type);
- candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) );
+ candidates.add(new CandidateResult(entityId, UUID.fromString(version)));
}
- CandidateResults candidateResults = new CandidateResults( query, candidates );
-
- if ( candidates.size() >= query.getLimit() ) {
- //USERGRID-461 our cursor is getting too large, map it to a new time UUID
- //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
-
- final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+ CandidateResults candidateResults = new CandidateResults(candidates);
+ if(candidateResults.hasCursor()) {
+ candidateResults.initializeCursor();
final String esScrollCursor = searchResponse.getScrollId();
-
//now set this into our map module
final int minutes = indexFig.getQueryCursorTimeout();
//just truncate it, we'll never hit a long value anyway
- mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds( minutes ) );
+ mapManager.putString(candidateResults.getCursor(), esScrollCursor, (int) TimeUnit.MINUTES.toSeconds(minutes));
- candidateResults.setCursor( userCursorString );
- logger.debug(" User cursor = {}, Cursor = {} ", userCursorString, esScrollCursor);
+ logger.debug(" User cursor = {}, Cursor = {} ", candidateResults.getCursor(), esScrollCursor);
}
return candidateResults;
}
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
new file mode 100644
index 0000000..d146e4c
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -0,0 +1,197 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createLegacyContextName;
+
+/**
+ * Classy class class.
+ */
+
+public class SearchRequestBuilderStrategy {
+
+ private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class);
+
+ private final EsProvider esProvider;
+ private final ApplicationScope applicationScope;
+ private final IndexIdentifier.IndexAlias alias;
+ private final int cursorTimeout;
+ public static final int MAX_LIMIT = 1000;
+
+ public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){
+
+ this.esProvider = esProvider;
+ this.applicationScope = applicationScope;
+ this.alias = alias;
+ this.cursorTimeout = cursorTimeout;
+ }
+
+ public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT);
+
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
+ .setTypes(searchTypes.getTypeNames(applicationScope))
+ .setScroll(cursorTimeout + "m")
+ .setQuery(createQueryBuilder( indexScope,query));
+
+ final FilterBuilder fb = createFilterBuilder(query);
+
+ //we have post filters, apply them
+ if (fb != null) {
+ logger.debug(" Filter: {} ", fb.toString());
+ srb = srb.setPostFilter(fb);
+ }
+
+
+ srb = srb.setFrom(0).setSize(limit);
+
+ for (Query.SortPredicate sp : query.getSortPredicates()) {
+
+ final SortOrder order;
+ if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) {
+ order = SortOrder.ASC;
+ } else {
+ order = SortOrder.DESC;
+ }
+
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
+
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(stringSort);
+
+ logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+
+ final String longFieldName = LONG_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(longSort);
+ logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
+
+
+ final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(doubleSort);
+ logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
+
+
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(booleanSort);
+ logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+ }
+ return srb;
+ }
+
+
+ public QueryBuilder createQueryBuilder( IndexScope indexScope, Query query) {
+ String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
+
+ QueryBuilder queryBuilder = null;
+
+ //we have a root operand. Translate our AST into an ES search
+ if ( query.getRootOperand() != null ) {
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
+ // will return a tile query + post filter
+ QueryVisitor v = new EsQueryVistor();
+
+ try {
+ query.getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+
+
+ queryBuilder = v.getQueryBuilder();
+ }
+
+
+ // Add our filter for context to our query for fast execution.
+ // Fast because it utilizes bitsets internally. See this post for more detail.
+ // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+
+ // TODO evaluate performance when it's an all query.
+ // Do we need to put the context term first for performance?
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for(String context : contexts){
+ boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
+ }
+ boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
+ if ( queryBuilder != null ) {
+ queryBuilder = boolQueryBuilder.must( queryBuilder );
+ }
+
+ //nothing was specified ensure we specify the context in the search
+ else {
+ queryBuilder = boolQueryBuilder;
+ }
+
+ return queryBuilder;
+ }
+
+
+ public FilterBuilder createFilterBuilder(Query query) {
+ FilterBuilder filterBuilder = null;
+
+ if ( query.getRootOperand() != null ) {
+ QueryVisitor v = new EsQueryVistor();
+ try {
+ query.getRootOperand().visit( v );
+
+ } catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+ filterBuilder = v.getFilterBuilder();
+ }
+
+ return filterBuilder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
index be90446..5cb6680 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
@@ -19,12 +19,14 @@ package org.apache.usergrid.persistence.index.query;
import java.util.Iterator;
import java.util.List;
+
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Internal results class, should not be returned as results to a user.
+ * Internal results class, should not be returned as results to a user.
* Only returns candidate entity results
*/
public class CandidateResults implements Iterable<CandidateResult> {
@@ -33,16 +35,25 @@ public class CandidateResults implements Iterable<CandidateResult> {
private String cursor = null;
- private final Query query;
private final List<CandidateResult> candidates;
- public CandidateResults( Query query, List<CandidateResult> candidates ) {
- this.query = query;
+ public CandidateResults( List<CandidateResult> candidates ) {
this.candidates = candidates;
}
+ public void add( List<CandidateResult> candidates ) {
+ this.candidates.addAll( candidates);
+ }
+
+ public void initializeCursor(){
+ //USERGRID-461 our cursor is getting too large, map it to a new time UUID
+ //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
+
+ cursor = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID());
+ }
+
public boolean hasCursor() {
return cursor != null;
@@ -59,11 +70,8 @@ public class CandidateResults implements Iterable<CandidateResult> {
}
- public Query getQuery() {
- return query;
- }
-
+
public int size() {
return candidates.size();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 438d82e..6454b48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -40,10 +40,7 @@ import org.antlr.runtime.Token;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
-import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
-import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -55,15 +52,10 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual;
import org.apache.usergrid.persistence.index.query.tree.LessThan;
import org.apache.usergrid.persistence.index.query.tree.LessThanEqual;
import org.apache.usergrid.persistence.index.query.tree.Operand;
-import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,9 +72,7 @@ public class Query {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
- public static final int DEFAULT_LIMIT = 10;
- public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
@@ -90,8 +80,6 @@ public class Query {
private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>();
private Operand rootOperand;
private UUID startResult;
- private String cursor;
- private int limit = 0;
private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
private boolean mergeSelectResults = false;
@@ -124,8 +112,6 @@ public class Query {
sortPredicates = q.sortPredicates != null
? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
selectAssignments = q.selectAssignments != null
? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
@@ -148,71 +134,6 @@ public class Query {
}
- public QueryBuilder createQueryBuilder( final String[] contexts ) {
-
-
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
-
- // Add our filter for context to our query for fast execution.
- // Fast because it utilizes bitsets internally. See this post for more detail.
- // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-
- // TODO evaluate performance when it's an all query.
- // Do we need to put the context term first for performance?
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- for(String context : contexts){
- boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
- }
- boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
- if ( queryBuilder != null ) {
- queryBuilder = boolQueryBuilder.must( queryBuilder );
- }
-
- //nothing was specified ensure we specify the context in the search
- else {
- queryBuilder = boolQueryBuilder;
- }
-
- return queryBuilder;
- }
-
-
- public FilterBuilder createFilterBuilder() {
- FilterBuilder filterBuilder = null;
-
- if ( getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
- try {
- getRootOperand().visit( v );
-
- } catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
- filterBuilder = v.getFilterBuilder();
- }
-
- return filterBuilder;
- }
/**
@@ -315,7 +236,6 @@ public class Query {
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
String cursor = ListUtils.first( params.get( "cursor" ) );
- Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
@@ -386,15 +306,7 @@ public class Query {
q.setStartResult( start );
}
- if ( cursor != null ) {
- q = newQueryIfNull( q );
- q.setCursor( cursor );
- }
- if ( limit != null ) {
- q = newQueryIfNull( q );
- q.setLimit( limit );
- }
if ( startTime != null ) {
q = newQueryIfNull( q );
@@ -437,22 +349,21 @@ public class Query {
public static Query searchForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query findForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
- q.setLimit( 1 );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID( uuid ) );
+ q.addIdentifier( Identifier.fromUUID(uuid) );
return q;
}
@@ -809,8 +720,8 @@ public class Query {
public Query addContainsFilter( String propName, String keyword ) {
ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) );
- equality.setProperty( propName );
- equality.setLiteral( keyword );
+ equality.setProperty(propName);
+ equality.setLiteral(keyword);
addClause( equality );
@@ -819,8 +730,8 @@ public class Query {
private void addClause( EqualityOperand equals, String propertyName, Object value ) {
- equals.setProperty( propertyName );
- equals.setLiteral( value );
+ equals.setProperty(propertyName);
+ equals.setLiteral(value);
addClause( equals );
}
@@ -882,7 +793,7 @@ public class Query {
}
- public UUID getStartResult() {
+ public UUID getStartResult(String cursor) {
if ( ( startResult == null ) && ( cursor != null ) ) {
byte[] cursorBytes = Base64.decodeBase64( cursor );
if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
@@ -893,61 +804,6 @@ public class Query {
}
- public String getCursor() {
- return cursor;
- }
-
-
- public void setCursor( String cursor ) {
- this.cursor = cursor;
- }
-
-
- public Query withCursor( String cursor ) {
- setCursor( cursor );
- return this;
- }
-
-
- public int getLimit() {
- return getLimit( DEFAULT_LIMIT );
- }
-
-
- public int getLimit( int defaultLimit ) {
- if ( limit <= 0 ) {
- if ( defaultLimit > 0 ) {
- return defaultLimit;
- }
- else {
- return DEFAULT_LIMIT;
- }
- }
- return limit;
- }
-
-
- public void setLimit( int limit ) {
-
- // TODO tnine. After users have had time to change their query limits,
- // this needs to be uncommented and enforced.
- // if(limit > MAX_LIMIT){
- // throw new IllegalArgumentException(
- // String.format("Query limit must be <= to %d", MAX_LIMIT));
- // }
-
- if ( limit > MAX_LIMIT ) {
- limit = MAX_LIMIT;
- }
-
- this.limit = limit;
- }
-
-
- public Query withLimit( int limit ) {
- setLimit( limit );
- return this;
- }
public boolean isReversed() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 6df5a6d..2c03635 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -118,7 +118,7 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
EntityIndexBatch batch = entityIndex.createBatch();
Entity entity = new Entity( entityType );
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
@@ -143,7 +143,7 @@ public class EntityIndexTest extends BaseIT {
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
}
@Test
@@ -219,17 +219,17 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
- ei.addIndex("v2", 1,0,"one");
+ ei.addIndex("v2", 1, 0, "one");
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 101, 100);
ei.refresh();
@@ -364,11 +364,12 @@ public class EntityIndexTest extends BaseIT {
StopWatch timer = new StopWatch();
timer.start();
Query query = Query.fromQL( queryString );
- query.setLimit( 1000 );
- CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query );
+ CandidateResults candidateResults = null;
+ candidateResults = entityIndex.search( scope, searchTypes, query,num+1 );
+
timer.stop();
- assertEquals( num, candidateResults.size() );
+ assertEquals( num,candidateResults.size() );
log.debug( "Query time {}ms", timer.getTime() );
return candidateResults;
}
@@ -652,13 +653,8 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
Query query = Query.fromQL( "select * order by created" );
- query.setLimit( limit );
-
- if ( cursor != null ) {
- query.setCursor( cursor );
- }
- final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(), query );
+ final CandidateResults results = cursor == null ? entityIndex.search( indexScope, SearchTypes.allTypes(), query , limit) : entityIndex.getNextPage(cursor);
assertTrue( results.hasCursor() );
@@ -667,8 +663,6 @@ public class EntityIndexTest extends BaseIT {
assertEquals("Should be 16 bytes as hex", 32, cursor.length());
-
-
assertEquals( 1, results.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 102e222..792fe3a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -19,6 +19,9 @@
package org.apache.usergrid.persistence.query.tree;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.impl.SearchRequestBuilderStrategy;
import org.apache.usergrid.persistence.index.query.tree.LongLiteral;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser;
@@ -43,6 +46,7 @@ import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Test;
import org.apache.usergrid.persistence.index.query.Query;
import org.elasticsearch.index.query.QueryBuilder;
@@ -429,7 +433,9 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"});
+ SearchRequestBuilderStrategy builderStrategy = new SearchRequestBuilderStrategy(null,new ApplicationScopeImpl(new SimpleId("test")),null,100);
+ QueryBuilder qb =builderStrategy.createQueryBuilder(new IndexScopeImpl(new SimpleId("owner"),"app"),query);
+
}
[14/24] incubator-usergrid git commit: refactor into observable
Posted by to...@apache.org.
refactor into observable
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7976e1d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7976e1d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7976e1d5
Branch: refs/heads/two-dot-o-dev
Commit: 7976e1d577d9ba3d5f096db33b6984fa5226b7eb
Parents: 2df4013
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 14:38:42 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 14:38:42 2015 -0600
----------------------------------------------------------------------
.../migration/EsIndexDataMigrationImpl.java | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7976e1d5/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 64b7d29..13e6526 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -33,6 +33,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder
import org.elasticsearch.client.AdminClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import rx.Observable;
import java.util.ArrayList;
import java.util.List;
@@ -64,21 +65,22 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
final AtomicInteger integer = new AtomicInteger();
- migrationDataProvider.getData().doOnNext(applicationScope -> {
- LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
- String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
- AdminClient adminClient = provider.getClient().admin();
+ final AdminClient adminClient = provider.getClient().admin();
- for (String index : indexes) {
+ migrationDataProvider.getData().flatMap(applicationScope -> {
+ LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig, applicationScope);
+ String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
+ return Observable.from(indexes);
+ })
+ .doOnNext(index -> {
IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases();
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
integer.incrementAndGet();
- }
- })
- .doOnError(error -> log.error("failed to migrate index",error))
- .toBlocking().last();
+ })
+ .doOnError(error -> log.error("failed to migrate index", error))
+ .toBlocking().lastOrDefault(null);
return dataVersion.getImplementationVersion();
}
[10/24] incubator-usergrid git commit: add version migration for
indexes
Posted by to...@apache.org.
add version migration for indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf0718e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf0718e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf0718e6
Branch: refs/heads/two-dot-o-dev
Commit: bf0718e611bb06d8bf901ba34f5d8172d5ef02bb
Parents: 208ed2b
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:53:20 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:53:20 2015 -0600
----------------------------------------------------------------------
.../AllApplicationsObservable.java | 3 +-
.../usergrid/corepersistence/CoreModule.java | 9 ++++-
.../rx/impl/AllApplicationsObservableImpl.java | 5 ++-
.../persistence/index/guice/IndexModule.java | 13 ++++++-
.../index/impl/EsEntityIndexImpl.java | 9 ++++-
.../migration/EsIndexDataMigrationImpl.java | 20 ++++++++--
.../index/migration/IndexDataVersions.java | 39 ++++++++++++++++++++
.../index/guice/TestIndexModule.java | 7 +++-
8 files changed, 94 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
index 1187cf2..055223e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/AllApplicationsObservable.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Application;
@@ -29,7 +30,7 @@ import rx.Observable;
/**
* Interface for generating an observable of all ApplicationScope
*/
-public interface AllApplicationsObservable {
+public interface AllApplicationsObservable extends MigrationDataProvider<ApplicationScope>{
/**
* Return all applications in our system
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 2e9b780..d994419 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.corepersistence;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.springframework.context.ApplicationContext;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
@@ -96,7 +97,13 @@ public class CoreModule extends AbstractModule {
AllNodesInGraphImpl.class );
}
} );
- install(new IndexModule());
+ install(new IndexModule(){
+ @Override
+ public void configureMigrationProvider() {
+ bind( new TypeLiteral<MigrationDataProvider<ApplicationScope>>() {} ).to(
+ AllApplicationsObservable.class );
+ }
+ });
// install(new MapModule()); TODO, re-enable when index module doesn't depend on queue
// install(new QueueModule());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 0fc5452..aa5b5dc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -138,5 +138,8 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
}
-
+ @Override
+ public Observable<ApplicationScope> getData() {
+ return getAllApplications();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index a42dea8..7ecce54 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -42,7 +42,7 @@ import org.apache.usergrid.persistence.queue.guice.QueueModule;
import org.safehaus.guicyfig.GuicyFigModule;
-public class IndexModule extends AbstractModule {
+public abstract class IndexModule extends AbstractModule {
@Override
protected void configure() {
@@ -75,8 +75,17 @@ public class IndexModule extends AbstractModule {
//wire up the collection migration plugin
Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class);
- }
+ //invoke the migration plugin config
+ configureMigrationProvider();
+ }
+
+ /**
+ * Gives callers the ability to to configure an instance of
+ *
+ * MigrationDataProvider<ApplicationScope> for providing data migrations
+ */
+ public abstract void configureMigrationProvider();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cab8ded..844260f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -27,10 +27,12 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.migration.IndexDataVersions;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
@@ -71,7 +73,7 @@ import java.util.*;
* Implements index using ElasticSearch Java API.
*/
@Singleton
-public class EsEntityIndexImpl implements AliasedEntityIndex {
+public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
@@ -442,6 +444,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return Health.RED;
}
+ @Override
+ public int getImplementationVersion() {
+ return IndexDataVersions.SINGLE_INDEX.getVersion();
+ }
+
/**
* Interface for operations.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index b5dab53..2a0bb15 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -20,6 +20,7 @@ import com.google.inject.Inject;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.migration.data.ProgressObserver;
+import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.AliasedEntityIndex;
import org.apache.usergrid.persistence.index.IndexAlias;
@@ -30,9 +31,12 @@ import org.apache.usergrid.persistence.index.impl.EsProvider;
import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Classy class class.
@@ -44,6 +48,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
private final IndexFig indexFig;
private final IndexIdentifier indexIdentifier;
private final EsIndexCache indexCache;
+ private final VersionedData dataVersion;
+ private static final Logger log = LoggerFactory.getLogger(EsIndexDataMigrationImpl.class);
@Inject
public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){
@@ -52,10 +58,12 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
this.indexCache = indexCache;
+ this.dataVersion = (VersionedData) entityIndex;
}
@Override
public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) {
+ final AtomicInteger integer = new AtomicInteger();
migrationDataProvider.getData().doOnNext(applicationScope -> {
LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope);
String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read);
@@ -66,19 +74,23 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ integer.incrementAndGet();
}
- });
- return 0;
+ })
+ .doOnError(error -> log.error("failed to migrate index",error))
+ .toBlocking().last();
+
+ return integer.get();
}
@Override
public boolean supports(int currentVersion) {
- return false;
+ return currentVersion < dataVersion.getImplementationVersion();
}
@Override
public int getMaxVersion() {
- return 0;
+ return dataVersion.getImplementationVersion();
}
/**
* Class is used to generate an index name and alias name the old way via app name
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
new file mode 100644
index 0000000..921d86a
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexDataVersions.java
@@ -0,0 +1,39 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.index.migration;
+
+/**
+ * Classy class class.
+ */
+public enum IndexDataVersions {
+ MANY_INDEXES(0),
+ SINGLE_INDEX(1);
+
+ private final int version;
+
+
+ private IndexDataVersions( final int version ) {this.version = version;}
+
+
+ public int getVersion() {
+ return version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf0718e6/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 4cf46d6..4e6700c 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -33,7 +33,12 @@ public class TestIndexModule extends TestModule {
install( new CommonModule());
// configure collections and our core astyanax framework
- install( new IndexModule() );
+ install( new IndexModule(){
+ @Override
+ public void configureMigrationProvider(){
+
+ }
+ });
install( new GuicyFigModule(IndexTestFig.class) );
}
}
[24/24] incubator-usergrid git commit: refactor initiaization index
logic
Posted by to...@apache.org.
refactor initiaization index logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/07cb9dfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/07cb9dfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/07cb9dfd
Branch: refs/heads/two-dot-o-dev
Commit: 07cb9dfd6563d974d8c6b9764633be44e389a4d0
Parents: 3a31836
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 11:45:15 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 11:45:15 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 14 +--------
.../index/ApplicationEntityIndex.java | 5 ----
.../usergrid/persistence/index/EntityIndex.java | 10 +++----
.../impl/EsApplicationEntityIndexImpl.java | 12 --------
.../index/impl/EsEntityIndexImpl.java | 13 +++++++++
.../migration/EsIndexDataMigrationImpl.java | 4 ++-
.../persistence/index/impl/EntityIndexTest.java | 30 +++++++++++---------
.../index/impl/IndexLoadTestsIT.java | 4 ---
8 files changed, 38 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3cfc1a4..99204a8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -164,9 +164,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.getApplication();
}
- ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
- applicationEntityIndex.initializeIndex();
+ entityIndex.initialize();
entityIndex.refresh();
@@ -265,10 +263,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
throw new ApplicationAlreadyExistsException( appName );
}
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application"));
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
-
getSetup().setupApplicationKeyspace( applicationId, appName );
final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
@@ -730,12 +724,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
EntityManager em = getEntityManager( appId );
- ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ));
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
- em.reindex(po);
-
em.reindex( po );
logger.info("\n\nRebuilt index for applicationId {} \n", appId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index cb1b6f0..86e97c5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -30,11 +30,6 @@ public interface ApplicationEntityIndex {
/**
- *
- */
- public void initializeIndex();
-
- /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 6783fb0..ff9008e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -48,28 +48,28 @@ public interface EntityIndex extends CPManager {
* @param replicas
* @param writeConsistency
*/
- public void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
+ void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
/**
* Refresh the index.
*/
- public void refresh();
+ void refresh();
/**
* Check health of cluster.
*/
- public Health getClusterHealth();
+ Health getClusterHealth();
/**
* Check health of this specific index.
*/
- public Health getIndexHealth();
-
+ Health getIndexHealth();
+ void initialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 9248980..84845d0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -116,18 +116,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public void initializeIndex() {
- final int numberOfShards = indexFig.getNumberOfShards();
- final int numberOfReplicas = indexFig.getNumberOfReplicas();
- indexCache.invalidate(alias);
- String[] indexes = entityIndex.getUniqueIndexes();
- if(indexes == null || indexes.length==0) {
- entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
- }
-
- }
-
- @Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 055af2a..ee28939 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -131,6 +131,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
}
@Override
+ public void initialize(){
+ final int numberOfShards = indexFig.getNumberOfShards();
+ final int numberOfReplicas = indexFig.getNumberOfReplicas();
+ aliasCache.invalidate(alias);
+ String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
+
+ if(reads.length==0 || writes.length==0) {
+ addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
+ }
+ }
+
+ @Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
//get index name with suffix attached
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 123527a..3706722 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -62,6 +62,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
final AdminClient adminClient = provider.getClient().admin();
final int latestVersion = dataVersion.getImplementationVersion();
+ entityIndex.initialize();
+
observer.start();
try {
migrationDataProvider.getData().flatMap(applicationScope -> {
@@ -74,7 +76,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
try {
- aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()).get();
} catch (InvalidAliasNameException e) {
log.debug("Failed to add alias due to name conflict",e);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 2c03635..27f0b9d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -30,9 +30,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.index.*;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
+import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,6 +85,10 @@ public class EntityIndexTest extends BaseIT {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Before
+ public void setup(){
+ ei.initialize();
+ }
@Test
public void testIndex() throws IOException, InterruptedException {
@@ -94,7 +96,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl(appId);
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl(appId, "things");
@@ -158,7 +160,7 @@ public class EntityIndexTest extends BaseIT {
final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
final IndexScope indexScope = new IndexScopeImpl(appId, "things");
final String entityType = "thing";
- entityIndex.initializeIndex();
+
final CountDownLatch latch = new CountDownLatch(threads);
final AtomicLong failTime=new AtomicLong(0);
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
@@ -201,9 +203,9 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
for(int i=0;i<10;i++) {
- entityIndex.initializeIndex();
+
}
}
@@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
@@ -328,7 +330,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
Map entityMap = new HashMap() {{
put( "name", "Ferrari 212 Inter" );
@@ -460,7 +462,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
@@ -509,7 +511,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
entityIndex.createBatch();
// Bill has favorites as string, age as string and retirement goal as number
@@ -581,9 +583,9 @@ public class EntityIndexTest extends BaseIT {
Id ownerId = new SimpleId( "multivaluedtype" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
- assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() );
+ assertEquals("index should be ready", Health.GREEN, ei.getIndexHealth());
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
ei.refresh();
assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
@@ -603,7 +605,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final EntityIndexBatch batch = entityIndex.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index 0275e53..b03f09d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -135,10 +135,6 @@ public class IndexLoadTestsIT extends BaseIT {
final IndexScope indexScope = new IndexScopeImpl( applicationId, "test" );
- final ApplicationEntityIndex appEntityIndex = entityIndexFactory.createApplicationEntityIndex( scope );
-
- appEntityIndex.initializeIndex();
-
batchWriteTPS = metricsFactory.getMeter( IndexLoadTestsIT.class, "write.tps" );
batchWriteTimer = metricsFactory.getTimer( IndexLoadTestsIT.class, "write.timer" );
[21/24] incubator-usergrid git commit: remove comment
Posted by to...@apache.org.
remove comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/33b1b69f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/33b1b69f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/33b1b69f
Branch: refs/heads/two-dot-o-dev
Commit: 33b1b69fc325c6c4e3bc2fe5628befb6bef2a43d
Parents: 32cb153
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 10:05:51 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 10:05:51 2015 -0600
----------------------------------------------------------------------
.../serialization/impl/migration/EdgeDataMigrationImpl.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/33b1b69f/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 411e318..2e0412f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -87,9 +87,7 @@ public class EdgeDataMigrationImpl implements DataMigration<GraphNode> {
final Observable<List<Edge>> observable = migrationDataProvider.getData().flatMap( graphNode -> {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
-
- //TODO: maybe move to retreive all indexes : https://github.com/elastic/elasticsearch/blob/master/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
- //https://www.google.com/webhp?sourceid=chrome-instant&ion=1&espv=2&ie=UTF-8#q=get%20all%20indexes%20elasticsearch
+
//get edges from the source
return edgesFromSourceObservable.edgesFromSource( gm, graphNode.entryNode ).buffer( 1000 )
.doOnNext( edges -> {
[09/24] incubator-usergrid git commit: merge from two-o-dev
Posted by to...@apache.org.
merge from two-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/208ed2b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/208ed2b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/208ed2b3
Branch: refs/heads/two-dot-o-dev
Commit: 208ed2b3c93771faf51a41903d617f67cc32b14a
Parents: f695724 6db7ce9
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Mar 26 10:20:30 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Mar 26 10:20:30 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 304 ++++++-------------
.../results/CollectionRefsVerifier.java | 44 +++
.../CollectionResultsLoaderFactoryImpl.java | 60 ++++
.../results/ConnectionRefsVerifier.java | 61 ++++
.../ConnectionResultsLoaderFactoryImpl.java | 65 ++++
.../results/ElasticSearchQueryExecutor.java | 212 +++++++++++++
.../corepersistence/results/QueryExecutor.java | 37 +++
.../corepersistence/results/RefsVerifier.java | 42 ---
.../results/ResultsLoaderFactory.java | 3 +-
.../results/ResultsLoaderFactoryImpl.java | 62 ----
.../persistence/MultiQueryIterator.java | 6 +-
.../apache/usergrid/persistence/Results.java | 26 +-
.../cassandra/QueryProcessorImpl.java | 12 +-
.../usergrid/persistence/PathQueryIT.java | 46 ++-
stack/core/src/test/resources/log4j.properties | 2 +
.../persistence/graph/GraphManagerIT.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 16 +-
.../index/impl/BufferQueueInMemoryImpl.java | 10 +-
.../index/impl/EsEntityIndexBatchImpl.java | 2 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 61 ++--
.../usergrid/persistence/index/query/Query.java | 55 ++--
.../persistence/index/utils/ListUtils.java | 3 +-
.../cassandra/ManagementServiceImpl.java | 18 +-
.../providers/PingIdentityProvider.java | 5 +-
24 files changed, 730 insertions(+), 424 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/208ed2b3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 8f8ac5e,90b8e35..f62648d
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -40,7 -65,7 +40,7 @@@ public class EsEntityIndexBatchImpl imp
private final ApplicationScope applicationScope;
-- private final IndexIdentifier.IndexAlias alias;
++ private final IndexAlias alias;
private final IndexIdentifier indexIdentifier;
private final IndexBufferProducer indexBatchBufferProducer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/208ed2b3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 6454b48,6362c0a..fa7def4
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@@ -106,31 -117,86 +106,39 @@@ public class Query
}
+ /**
+ * Creates a deep copy of a query from another query
+ * @param q
+ */
public Query( Query q ) {
- if ( q != null ) {
- type = q.type;
- sortPredicates = q.sortPredicates != null
- ? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
- startResult = q.startResult;
- selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
- mergeSelectResults = q.mergeSelectResults;
- //level = q.level;
- connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<String>( q.permissions ) : null;
- reversed = q.reversed;
- reversedSet = q.reversedSet;
- startTime = q.startTime;
- finishTime = q.finishTime;
- resolution = q.resolution;
- pad = q.pad;
- rootOperand = q.rootOperand;
- identifiers = q.identifiers != null
- ? new ArrayList<Identifier>( q.identifiers ) : null;
- counterFilters = q.counterFilters != null
- ? new ArrayList<CounterFilterPredicate>( q.counterFilters ) : null;
- collection = q.collection;
- if ( q == null ) {
++ if (q == null) {
+ return;
}
-
+ type = q.type;
+ sortPredicates = q.sortPredicates != null
- ? new ArrayList<>( q.sortPredicates ) : null;
++ ? new ArrayList<SortPredicate>(q.sortPredicates) : null;
+ startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
+ selectAssignments = q.selectAssignments != null
- ? new LinkedHashMap<>( q.selectAssignments ) : null;
++ ? new LinkedHashMap<String, String>(q.selectAssignments) : null;
+ mergeSelectResults = q.mergeSelectResults;
+ //level = q.level;
+ connectionType = q.connectionType;
- permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
++ permissions = q.permissions != null ? new ArrayList<String>(q.permissions) : null;
+ reversed = q.reversed;
+ reversedSet = q.reversedSet;
+ startTime = q.startTime;
+ finishTime = q.finishTime;
+ resolution = q.resolution;
+ pad = q.pad;
+ rootOperand = q.rootOperand;
+ identifiers = q.identifiers != null
- ? new ArrayList<>( q.identifiers ) : null;
++ ? new ArrayList<Identifier>(q.identifiers) : null;
+ counterFilters = q.counterFilters != null
- ? new ArrayList<>( q.counterFilters ) : null;
++ ? new ArrayList<CounterFilterPredicate>(q.counterFilters) : null;
+ collection = q.collection;
- level = q.level;
-
- }
-
-
- public QueryBuilder createQueryBuilder( final String context ) {
-
-
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
-
- // Add our filter for context to our query for fast execution.
- // Fast because it utilizes bitsets internally. See this post for more detail.
- // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-
- // TODO evaluate performance when it's an all query.
- // Do we need to put the context term first for performance?
- if ( queryBuilder != null ) {
- queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
- .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
- }
+
- //nothing was specified ensure we specify the context in the search
- else {
- queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- }
++ level = q.level;
+
- return queryBuilder;
}
@@@ -235,7 -317,8 +243,6 @@@
Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
-- String cursor = ListUtils.first( params.get( "cursor" ) );
- Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );