You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/04/18 00:48:12 UTC
[67/68] [abbrv] incubator-usergrid git commit: check trailing page
check trailing page
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1f83f0fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1f83f0fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1f83f0fa
Branch: refs/heads/two-dot-o-dev
Commit: 1f83f0fabcac3281024637300978ff8eb7ec366e
Parents: 0db7f4c
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Apr 17 16:43:31 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Apr 17 16:43:31 2015 -0600
----------------------------------------------------------------------
.../query/OrderByDiscardPagesPagingIT.java | 164 -------------------
.../persistence/index/CandidateResults.java | 9 +-
.../impl/EsApplicationEntityIndexImpl.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 17 +-
4 files changed, 20 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
deleted file mode 100644
index 2cee402..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/OrderByDiscardPagesPagingIT.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence.query;
-
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.index.query.Query;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.cassandra.QueryProcessor;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-/**
- *
- */
-public class OrderByDiscardPagesPagingIT extends IteratingQueryIT {
-
- private static final Logger LOG = LoggerFactory.getLogger( OrderByDiscardPagesPagingIT.class );
-
- private static final String search = "select * where field1 = true AND field2 = true order by ordinal";
-
- private static final int PAGE_SIZE = 300;
-
-
- @Test
- public void testUnionPagingCollection() throws Exception {
-
-
- final CollectionIoHelper collectionIoHelper = new CollectionIoHelper( app );
-
- Set<UUID> created = performSetup( collectionIoHelper );
-
-
- testUnionPaging( collectionIoHelper, search, created );
- }
-
-
- @Test
- public void testUnionPagingConnection() throws Exception {
-
- final ConnectionHelper connectionHelper = new ConnectionHelper( app );
-
- Set<UUID> created = performSetup( connectionHelper );
-
-
- testUnionPaging( connectionHelper, search, created );
- }
-
-
- private Set<UUID> performSetup( final IoHelper io ) throws Exception {
- io.doSetup();
-
- int size = ( int ) ( QueryProcessor.PAGE_SIZE * 2.5 );
-
- long start = System.currentTimeMillis();
-
- LOG.info( "Writing {} entities.", size );
-
- Set<UUID> entites = new HashSet<UUID>( size );
-
- for ( int i = 0; i < size; i++ ) {
-
- Map<String, Object> entity = new HashMap<String, Object>();
-
- entity.put( "ordinal", i );
-
- int segment = i / PAGE_SIZE;
-
- boolean shouldBeReturned = segment % 2 != 0;
-
- if ( shouldBeReturned ) {
-
- entity.put( "field1", true );
- entity.put( "field2", true );
- }
- else {
- entity.put( "field1", false );
- entity.put( "field2", false );
- }
-
-
- Entity saved = io.writeEntity( entity );
-
- LOG.info( "Writing entity with id '{}'", saved.getUuid() );
-
- if ( shouldBeReturned ) {
- entites.add( saved.getUuid() );
- }
- }
-
- long stop = System.currentTimeMillis();
-
- LOG.info( "Writes took {} ms", stop - start );
-
- return entites ;
- }
-
-
- private void testUnionPaging( final IoHelper io, final String queryString, final Set<UUID> expectedResults )
- throws Exception {
-
- //our field1Or has a result size < our page size, so it shouldn't blow up when the cursor is getting created
- //the leaf iterator should insert it's own "no value left" into the cursor
- Query query = Query.fromQL( queryString );
- query.setLimit( PAGE_SIZE );
-
- Results results;
-
- long start = System.currentTimeMillis();
-
- do {
-
- // now do simple ordering, should be returned in order
- results = io.getResults( query );
-
- final List<Entity> entities = results.getEntities();
-
- for ( int i = 0; i < entities.size(); i++ ) {
- final UUID uuid = entities.get( i ).getUuid();
-
- assertTrue( "Value should not be returned twice", expectedResults.contains( uuid ) );
-
- expectedResults.remove( uuid );
- }
-
- query.setCursor( results.getCursor() );
- }
- while ( results.getCursor() != null );
-
- long stop = System.currentTimeMillis();
-
- LOG.info( "Query took {} ms to return {} entities", stop - start, expectedResults.size() );
-
- assertEquals( "All entities returned", 0, expectedResults.size() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index f0955ea..14130bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -49,12 +49,16 @@ public class CandidateResults implements Iterable<CandidateResult> {
private final List<CandidateResult> candidates;
private final Collection<SelectFieldMapping> getFieldMappings;
+ private final String esCursor;
public CandidateResults( List<CandidateResult> candidates,
- final Collection<SelectFieldMapping> getFieldMappings ) {
+ final Collection<SelectFieldMapping> getFieldMappings,
+ final String esCursor
+ ) {
this.candidates = candidates;
this.getFieldMappings = getFieldMappings;
+ this.esCursor = esCursor;
}
@@ -109,4 +113,7 @@ public class CandidateResults implements Iterable<CandidateResult> {
return candidates.iterator();
}
+ public String getEsCursor() {
+ return esCursor;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 6574985..5b05b27 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -310,8 +310,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
candidates.add( candidateResult );
}
- final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings());
final String esScrollCursor = searchResponse.getScrollId();
+ final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor);
// >= seems odd. However if we get an overflow, we need to account for it.
if ( esScrollCursor != null && length >= limit ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1f83f0fa/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index e95c2bf..9b8a9f5 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -641,6 +641,7 @@ public class EntityIndexTest extends BaseIT {
@Test
public void testCursorFormat() throws Exception {
+ String myType = UUID.randomUUID().toString();
Id appId = new SimpleId( "application" );
Id ownerId = new SimpleId( "owner" );
@@ -655,7 +656,7 @@ public class EntityIndexTest extends BaseIT {
final EntityIndexBatch batch = entityIndex.createBatch();
- final int size = 10;
+ final int size = 100;
final List<Id> entityIds = new ArrayList<>( size );
@@ -671,6 +672,7 @@ public class EntityIndexTest extends BaseIT {
put( "email", "ed@anuff.com" );
put( "middlename", middleName );
put( "ordinal", ordinal );
+ put( "mytype", myType);
}};
final Id userId = new SimpleId( "user" );
@@ -691,17 +693,18 @@ public class EntityIndexTest extends BaseIT {
ei.refreshAsync().toBlocking().last();
- final int limit = 1;
+ final int limit = 5;
final int expectedPages = size / limit;
String cursor = null;
+ UUID lastId = null;
for ( int i = 0; i < expectedPages; i++ ) {
//**
- final String query = "select * order by ordinal asc";
+ final String query = "select * where mytype='"+myType+"' order by ordinal asc";
final CandidateResults results =
cursor == null ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit ) :
@@ -713,10 +716,12 @@ public class EntityIndexTest extends BaseIT {
assertEquals( "Should be 16 bytes as hex", 32, cursor.length() );
- assertEquals( 1, results.size() );
+ assertEquals(limit, results.size());
-
- assertEquals( results.get( 0 ).getId(), entityIds.get( i ) );
+ int ordinal = 0;//i == 0 ? 0 : 1;
+ assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
+ lastId = results.get((i+1)*limit-1).getId().getUuid();
+ assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get( i*limit ) );
}
//get our next page, we shouldn't get a cursor