You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ji...@apache.org on 2016/07/05 23:15:31 UTC
[08/32] incubator-geode git commit: GEODE-11: Pagination of Lucene
Query Results after entries are destroyed from the region
GEODE-11: Pagination of Lucene Query Results after entries are destroyed from the region
If a region entry is destroyed after a query is executed, the pages.next() call returns
a null value for the deleted entry. The code is changed to ignore deleted entries and
fetch the next available entry so that the page count remains consistent. Added tests to
verify scenarios of single, multiple and all region entry deletions.
Added a test to verify that all entries are fetched in one page when pagination is
disabled.
Signed-off-by: Anilkumar Gingade <ag...@pivotal.io>
This closes #178
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/1a3c38af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/1a3c38af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/1a3c38af
Branch: refs/heads/feature/GEODE-1571
Commit: 1a3c38af10ad0b3dc0e6e9222a2c44f4f6598a3e
Parents: 6a9a83c
Author: Aparna Dharmakkan <ad...@pivotal.io>
Authored: Wed Jun 29 16:06:38 2016 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Fri Jul 1 11:46:40 2016 -0700
----------------------------------------------------------------------
.../PageableLuceneQueryResultsImpl.java | 66 ++++++----
.../lucene/LuceneQueriesIntegrationTest.java | 119 +++++++++++++++++--
2 files changed, 158 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1a3c38af/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
index 17668a7..1563df0 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/PageableLuceneQueryResultsImpl.java
@@ -20,6 +20,7 @@
package com.gemstone.gemfire.cache.lucene.internal;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -42,7 +43,11 @@ public class PageableLuceneQueryResultsImpl<K,V> implements PageableLuceneQueryR
* list of docs matching search query
*/
private final List<EntryScore<K>> hits;
-
+
+ /**
+ * * Current page of results
+ */
+ private List<LuceneResultStruct<K,V>> currentPage;
/**
* The maximum score. Lazily evaluated
*/
@@ -69,38 +74,59 @@ public class PageableLuceneQueryResultsImpl<K,V> implements PageableLuceneQueryR
this.pageSize = pageSize == 0 ? Integer.MAX_VALUE : pageSize;
}
- @Override
- public List<LuceneResultStruct<K,V>> next() {
- if(!hasNext()) {
- throw new NoSuchElementException();
- }
-
- int end = currentHit + pageSize;
- end = end > hits.size() ? hits.size() : end;
- List<EntryScore<K>> scores = hits.subList(currentHit, end);
-
- ArrayList<K> keys = new ArrayList<K>(hits.size());
+
+ public List<LuceneResultStruct<K,V>> getHitEntries(int fromIndex, int toIndex) {
+ List<EntryScore<K>> scores = hits.subList(fromIndex, toIndex);
+ ArrayList<K> keys = new ArrayList<K>(scores.size());
for(EntryScore<K> score : scores) {
keys.add(score.getKey());
}
-
+
Map<K,V> values = userRegion.getAll(keys);
- ArrayList<LuceneResultStruct<K,V>> results = new ArrayList<LuceneResultStruct<K,V>>(hits.size());
+ ArrayList<LuceneResultStruct<K,V>> results = new ArrayList<LuceneResultStruct<K,V>>(scores.size());
for(EntryScore<K> score : scores) {
V value = values.get(score.getKey());
- results.add(new LuceneResultStructImpl(score.getKey(), value, score.getScore()));
+ if (value!=null)
+ results.add(new LuceneResultStructImpl(score.getKey(), value, score.getScore()));
}
-
-
- currentHit = end;
-
return results;
}
@Override
+ public List<LuceneResultStruct<K,V>> next() {
+ if(!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ List<LuceneResultStruct<K,V>> result = advancePage();
+ currentPage = null;
+ return result;
+ }
+
+ private List<LuceneResultStruct<K, V>> advancePage() {
+ if(currentPage != null) {
+ return currentPage;
+ }
+
+ int resultSize = (pageSize != Integer.MAX_VALUE) ? pageSize : hits.size();
+ currentPage = new ArrayList<LuceneResultStruct<K,V>>(resultSize);
+ while (currentPage.size()<pageSize && currentHit < hits.size()) {
+ int end = currentHit + pageSize - currentPage.size();
+ end = end > hits.size() ? hits.size() : end;
+ currentPage.addAll(getHitEntries(currentHit, end));
+ currentHit = end;
+ }
+ return currentPage;
+ }
+
+ @Override
public boolean hasNext() {
- return hits.size() > currentHit;
+
+ advancePage();
+ if ( currentPage.isEmpty() ) {
+ return false;
+ }
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1a3c38af/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
index 988e4f5..ccd6d41 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
@@ -35,6 +36,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.contrib.java.lang.system.SystemOutRule;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
@@ -43,6 +45,7 @@ import com.gemstone.gemfire.cache.RegionShortcut;
import com.gemstone.gemfire.cache.lucene.test.TestObject;
import com.gemstone.gemfire.pdx.JSONFormatter;
import com.gemstone.gemfire.pdx.PdxInstance;
+import com.gemstone.gemfire.pdx.internal.AutoSerializableManager.ObjectArrayField;
import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
/**
@@ -119,28 +122,32 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest {
@Test()
public void shouldPaginateResults() throws Exception {
- final LuceneQuery<Object, Object> query = addValuesAndCreateQuery();
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
assertTrue(pages.hasNext());
- assertEquals(3, pages.size());
+ assertEquals(7, pages.size());
final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
final List<LuceneResultStruct<Object, Object>> page2 = pages.next();
+ final List<LuceneResultStruct<Object, Object>> page3 = pages.next();
+ final List<LuceneResultStruct<Object, Object>> page4 = pages.next();
List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
allEntries.addAll(page1);
allEntries.addAll(page2);
+ allEntries.addAll(page3);
+ allEntries.addAll(page4);
assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
-
}
+
@Test
public void shouldReturnValuesFromFindValues() throws Exception {
- final LuceneQuery<Object, Object> query = addValuesAndCreateQuery();
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
assertEquals(region.values(), new HashSet(query.findValues()));
}
- private LuceneQuery<Object, Object> addValuesAndCreateQuery() {
+ private LuceneQuery<Object, Object> addValuesAndCreateQuery(int pagesize) {
luceneService.createIndex(INDEX_NAME, REGION_NAME, "field1", "field2");
region = cache.createRegionFactory(RegionShortcut.PARTITION)
.create(REGION_NAME);
@@ -153,10 +160,14 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest {
region.put("A", new TestObject(value1, value1));
region.put("B", new TestObject(value2, value2));
region.put("C", new TestObject(value3, value3));
+ region.put("D", new TestObject(value1, value1));
+ region.put("E", new TestObject(value2, value2));
+ region.put("F", new TestObject(value3, value3));
+ region.put("G", new TestObject(value1, value2));
index.waitUntilFlushed(60000);
return luceneService.createLuceneQueryFactory()
- .setPageSize(2)
+ .setPageSize(pagesize)
.create(INDEX_NAME, REGION_NAME,
"one", "field1");
}
@@ -260,7 +271,101 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest {
thrown.expect(LuceneQueryException.class);
query.findPages();
}
-
+
+ @Test
+ public void shouldReturnAllResultsWhenPaginationIsDisabled() throws Exception {
+ // Pagination disabled by setting page size = 0.
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(0);
+ final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+ assertTrue(pages.hasNext());
+ assertEquals(7, pages.size());
+ final List<LuceneResultStruct<Object, Object>> page = pages.next();
+ assertFalse(pages.hasNext());
+ assertEquals(region.keySet(), page.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+ assertEquals(region.values(), page.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+ }
+
+ @Test
+ public void shouldReturnCorrectResultsOnDeletionAfterQueryExecution() throws Exception {
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+ final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+ List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
+ assertTrue(pages.hasNext());
+ assertEquals(7, pages.size());
+ // Destroying an entry from the region after the query is executed.
+ region.destroy("C");
+ final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
+ assertEquals(2, page1.size());
+ final List<LuceneResultStruct<Object, Object>> page2 = pages.next();
+ assertEquals(2, page2.size());
+ final List<LuceneResultStruct<Object, Object>> page3 = pages.next();
+ assertEquals(2, page3.size());
+ assertFalse(pages.hasNext());
+
+ allEntries.addAll(page1);
+ allEntries.addAll(page2);
+ allEntries.addAll(page3);
+ assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+ assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+ }
+
+ @Test
+ public void shouldReturnCorrectResultsOnMultipleDeletionsAfterQueryExecution() throws Exception {
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+
+ final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+ List<LuceneResultStruct<Object, Object>> allEntries=new ArrayList<>();
+
+ assertTrue(pages.hasNext());
+ assertEquals(7, pages.size());
+
+ // Destroying an entry from the region after the query is executed.
+ region.destroy("C");
+ allEntries.addAll(pages.next());
+
+ // Destroying an entry from allEntries and the region after it is fetched through pages.next().
+ Object removeKey = ((LuceneResultStruct)allEntries.remove(0)).getKey();
+ region.destroy(removeKey);
+ allEntries.addAll(pages.next());
+
+ // Destroying a region entry which has't been fetched through pages.next() yet.
+ Set resultKeySet = allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet());
+
+ for(Object key : region.keySet()) {
+ if (!resultKeySet.contains(key)) {
+ region.destroy(key);
+ break;
+ }
+ }
+
+ allEntries.addAll(pages.next());
+ assertFalse(pages.hasNext());
+
+ assertEquals(region.keySet(), allEntries.stream().map(entry -> entry.getKey()).collect(Collectors.toSet()));
+ assertEquals(region.values(), allEntries.stream().map(entry -> entry.getValue()).collect(Collectors.toSet()));
+ }
+
+ @Test
+ public void shouldReturnCorrectResultsOnAllDeletionsAfterQueryExecution() throws Exception {
+ final LuceneQuery<Object, Object> query = addValuesAndCreateQuery(2);
+
+ final PageableLuceneQueryResults<Object, Object> pages = query.findPages();
+ assertTrue(pages.hasNext());
+ assertEquals(7, pages.size());
+ region.destroy("A");
+ region.destroy("B");
+ region.destroy("C");
+ region.destroy("D");
+ region.destroy("E");
+ region.destroy("F");
+ region.destroy("G");
+ assertTrue(pages.hasNext());
+ final List<LuceneResultStruct<Object, Object>> page1 = pages.next();
+ assertEquals(2, page1.size());
+ assertFalse(pages.hasNext());
+
+ }
+
private PdxInstance insertAJson(Region region, String key) {
String jsonCustomer = "{"
+ "\"name\": \""+key+"\","