You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by lu...@apache.org on 2018/03/23 17:40:36 UTC
asterixdb git commit: [NO ISSUE][STO] Misc fixes of inverted index
Repository: asterixdb
Updated Branches:
refs/heads/master f5257fef5 -> 8aa13cacf
[NO ISSUE][STO] Misc fixes of inverted index
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch mainly improves the inverted index merge perf
in the following ways:
- Fix the bloom filter check on deleted btree
- Remove unnecessary btree search during inverted index range search
- Remove unnecessary page pins when merging inverted lists by
making the cursor remembering the last pinned page
- Fix close/destroy operations of range search cursor
Change-Id: I7bbf6fe2d332db0e0368a4ce0f1ab218ba6da81d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2514
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/8aa13cac
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/8aa13cac
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/8aa13cac
Branch: refs/heads/master
Commit: 8aa13cacfbbb5c22b4bf48515c48a5b96853ce02
Parents: f5257fe
Author: luochen01 <cl...@uci.edu>
Authored: Thu Mar 22 19:26:00 2018 -0700
Committer: Luo Chen <cl...@uci.edu>
Committed: Fri Mar 23 10:40:12 2018 -0700
----------------------------------------------------------------------
.../tuples/ConcatenatingTupleReference.java | 17 +--
.../lsm/common/impls/LSMIndexSearchCursor.java | 3 +-
.../dataflow/LSMInvertedIndexLocalResource.java | 4 +-
.../invertedindex/impls/LSMInvertedIndex.java | 106 +++++++++----------
.../LSMInvertedIndexRangeSearchCursor.java | 30 +++++-
.../impls/PartitionedLSMInvertedIndex.java | 5 +-
.../FixedSizeElementInvertedListScanCursor.java | 22 ++--
.../ondisk/OnDiskInvertedIndex.java | 10 +-
.../ondisk/OnDiskInvertedIndexOpContext.java | 15 ++-
.../OnDiskInvertedIndexRangeSearchCursor.java | 38 ++-----
.../ondisk/PartitionedOnDiskInvertedIndex.java | 2 +-
.../invertedindex/util/InvertedIndexUtils.java | 9 +-
.../util/LSMInvertedIndexTestContext.java | 11 +-
13 files changed, 151 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
index 18de540..c635411 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/ConcatenatingTupleReference.java
@@ -19,8 +19,6 @@
package org.apache.hyracks.storage.am.common.tuples;
-import java.util.Arrays;
-
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
/**
@@ -96,14 +94,17 @@ public class ConcatenatingTupleReference implements ITupleReference {
return tuples[tupleIndex].getFieldLength(fieldIndex);
}
+ /**
+ * Right now this class is only used by inverted index, and only contains 2 tuples.
+ * As a result, sequential search would be more efficient than binary search
+ */
private int getTupleIndex(int fIdx) {
- int tupleIndex = Arrays.binarySearch(fieldCounts, 0, numTuples - 1, fIdx);
- if (tupleIndex < 0) {
- tupleIndex = -tupleIndex - 1;
- } else {
- ++tupleIndex;
+ for (int i = 0; i < numTuples; i++) {
+ if (fIdx < fieldCounts[i]) {
+ return i;
+ }
}
- return tupleIndex;
+ throw new IllegalArgumentException("Illegal field index " + fIdx);
}
private int getFieldIndex(int tupleIndex, int fIdx) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 445a005..fb1fa63 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -125,7 +125,6 @@ public abstract class LSMIndexSearchCursor extends EnforcedIndexCursor implement
rangeCursors[i].close();
}
}
- rangeCursors = null;
} finally {
if (lsmHarness != null) {
lsmHarness.endSearch(opCtx);
@@ -249,7 +248,7 @@ public abstract class LSMIndexSearchCursor extends EnforcedIndexCursor implement
public class PriorityQueueElement {
private ITupleReference tuple;
- private int cursorIndex;
+ private final int cursorIndex;
public PriorityQueueElement(int cursorIndex) {
tuple = null;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index 4eb7728..f2fb9af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -95,14 +95,14 @@ public class LSMInvertedIndexLocalResource extends LsmResource {
opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory,
invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields,
filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
} else {
return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this),
ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories,
filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable,
- metadataPageManagerFactory);
+ metadataPageManagerFactory, serviceCtx.getTracer());
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 5fda514..e46c24a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -100,10 +100,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
opTracker, ioScheduler, ioOpCallbackFactory, componentFactory, componentFactory, filterFrameFactory,
- filterManager, filterFields, durable, filterHelper, invertedIndexFields, ITracer.NONE);
+ filterManager, filterFields, durable, filterHelper, invertedIndexFields, tracer);
this.tokenizerFactory = tokenizerFactory;
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
@@ -349,65 +349,65 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
IIndexCursor cursor = mergeOp.getCursor();
- try {
- ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
- // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
- // Create an inverted index instance.
- ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(),
- mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- ILSMDiskComponentBulkLoader componentBulkLoader;
- // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted
- // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged.
- if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- // Keep the deleted tuples since the oldest disk component is not included in the merge operation
- LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
- new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
- try {
- long numElements = 0L;
- for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i))
- .getBloomFilter().getNumElements();
- }
- componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false);
- loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader);
- } finally {
- btreeCursor.destroy();
- }
- } else {
- componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false);
- }
- search(opCtx, cursor, mergePred);
+ ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
+ // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
+ // Create an inverted index instance.
+ ILSMDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
+ ILSMDiskComponentBulkLoader componentBulkLoader;
+ // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the inverted
+ // indexes so that lsmHarness.endSearch() is called once when the inverted indexes have been merged.
+ if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
+ .get(diskComponents.size() - 1)) {
+ // Keep the deleted tuples since the oldest disk component is not included in the merge operation
+ LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor =
+ new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference tuple = cursor.getTuple();
- componentBulkLoader.add(tuple);
+ long numElements = 0L;
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i))
+ .getBloomFilter().getNumElements();
}
+ componentBulkLoader = component.createBulkLoader(1.0f, false, numElements, false, false, false);
+ loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader);
} finally {
+ btreeCursor.destroy();
+ }
+ } else {
+ componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false);
+ }
+ search(opCtx, cursor, mergePred);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ componentBulkLoader.add(cursor.getTuple());
+ }
+ } finally {
+ try {
cursor.close();
+ } finally {
+ cursor.destroy();
}
- if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
- for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
- ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
- if (min != null) {
- filterTuples.add(min);
- }
- if (max != null) {
- filterTuples.add(max);
- }
+ }
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
+ ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
+ if (min != null) {
+ filterTuples.add(min);
+ }
+ if (max != null) {
+ filterTuples.add(max);
}
- getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples,
- NoOpOperationCallback.INSTANCE);
- getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
}
- componentBulkLoader.end();
- return component;
- } finally {
- cursor.close();
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples,
+ NoOpOperationCallback.INSTANCE);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
}
+ componentBulkLoader.end();
+
+ return component;
}
private void loadDeleteTuples(ILSMIndexOperationContext opCtx,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index 12dc23f..020b788 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -98,10 +98,9 @@ public class LSMInvertedIndexRangeSearchCursor extends LSMIndexSearchCursor {
keysOnlyTuple.reset(checkElement.getTuple());
int end = checkElement.getCursorIndex();
for (int i = 0; i < end; i++) {
- if (bloomFilters[i] != null && bloomFilters[i].contains(keysOnlyTuple, hashes)) {
+ if (bloomFilters[i] != null && !bloomFilters[i].contains(keysOnlyTuple, hashes)) {
continue;
}
- deletedKeysBTreeCursors[i].close();
deletedKeysBTreeAccessors.get(i).search(deletedKeysBTreeCursors[i], keySearchPred);
try {
if (deletedKeysBTreeCursors[i].hasNext()) {
@@ -114,4 +113,31 @@ public class LSMInvertedIndexRangeSearchCursor extends LSMIndexSearchCursor {
return false;
}
+ @Override
+ public void doClose() throws HyracksDataException {
+ try {
+ super.doClose();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].close();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void doDestroy() throws HyracksDataException {
+ try {
+ super.doDestroy();
+ } finally {
+ if (deletedKeysBTreeCursors != null) {
+ for (int i = 0; i < deletedKeysBTreeCursors.length; i++) {
+ deletedKeysBTreeCursors[i].destroy();
+ }
+ deletedKeysBTreeCursors = null;
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
index d7482a8..604d374 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java
@@ -40,6 +40,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.inmemory.InMemoryInverted
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.util.InvertedIndexUtils;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
@@ -52,12 +53,12 @@ public class PartitionedLSMInvertedIndex extends LSMInvertedIndex {
IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps,
- int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) throws HyracksDataException {
+ int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, ITracer tracer) throws HyracksDataException {
super(ioManager, virtualBufferCaches, componentFactory, filterHelper, filterFrameFactory, filterManager,
bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits, invListCmpFactories,
tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler,
ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
index b033b99..2401c67 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListScanCursor.java
@@ -53,6 +53,7 @@ public class FixedSizeElementInvertedListScanCursor extends InvertedListCursor {
protected ICachedPage page;
protected boolean pinned;
+ protected int pinnedPageId = -1;
public FixedSizeElementInvertedListScanCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields)
throws HyracksDataException {
@@ -103,14 +104,18 @@ public class FixedSizeElementInvertedListScanCursor extends InvertedListCursor {
*/
@Override
public void loadPages() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
if (currentPageId == endPageId) {
+ // inverted list exhausted, return
return;
}
currentPageId++;
+ if (pinned && pinnedPageId == currentPageId) {
+ // already pinned, return
+ return;
+ }
+ unloadPages();
page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
+ pinnedPageId = currentPageId;
pinned = true;
}
@@ -134,7 +139,6 @@ public class FixedSizeElementInvertedListScanCursor extends InvertedListCursor {
// Deducts 1 since the startPage would be set to bufferCurrentPageId + 1 in loadPages().
this.currentPageId = startPageId - 1;
this.numPages = endPageId - startPageId + 1;
- this.pinned = false;
}
@Override
@@ -158,16 +162,14 @@ public class FixedSizeElementInvertedListScanCursor extends InvertedListCursor {
@Override
public void doClose() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ // No op
+ // We allow the inverted list cursor to hold at most one page to avoid
+ // unnecessary pins
}
@Override
public void doDestroy() throws HyracksDataException {
- if (pinned) {
- unloadPages();
- }
+ unloadPages();
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 2f4f1d6..14ebe46 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -204,9 +204,9 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
try {
if (ctx.getBtreeCursor().hasNext()) {
ctx.getBtreeCursor().next();
- openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor);
+ openInvertedListCursor(ctx.getBtreeCursor().getTuple(), listCursor, ctx);
} else {
- LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState = ctx.getCursorInitialState();
initState.setInvertedListInfo(0, 0, 0, 0);
listCursor.open(initState, null);
}
@@ -215,8 +215,8 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
}
}
- public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor)
- throws HyracksDataException {
+ public void openInvertedListCursor(ITupleReference btreeTuple, InvertedListCursor listCursor,
+ OnDiskInvertedIndexOpContext opCtx) throws HyracksDataException {
int startPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListStartPageIdField),
btreeTuple.getFieldStart(invListStartPageIdField));
int endPageId = IntegerPointable.getInteger(btreeTuple.getFieldData(invListEndPageIdField),
@@ -225,7 +225,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
btreeTuple.getFieldStart(invListStartOffField));
int numElements = IntegerPointable.getInteger(btreeTuple.getFieldData(invListNumElementsField),
btreeTuple.getFieldStart(invListNumElementsField));
- LSMInvertedIndexSearchCursorInitialState initState = new LSMInvertedIndexSearchCursorInitialState();
+ LSMInvertedIndexSearchCursorInitialState initState = opCtx.getCursorInitialState();
initState.setInvertedListInfo(startPageId, endPageId, startOff, numElements);
listCursor.open(initState, null);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
index 267cc79..81bc46b9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexSearchCursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
@@ -32,12 +33,13 @@ import org.apache.hyracks.storage.common.MultiComparator;
public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
private final RangePredicate btreePred = new RangePredicate(null, null, true, true, null, null);
- private IIndexAccessor btreeAccessor;
- private IIndexCursor btreeCursor;
- private MultiComparator searchCmp;
+ private final IIndexAccessor btreeAccessor;
+ private final IIndexCursor btreeCursor;
+ private final MultiComparator searchCmp;
// For prefix search on partitioned indexes.
private MultiComparator prefixSearchCmp;
private boolean destroyed = false;
+ private LSMInvertedIndexSearchCursorInitialState cursorInitialState;
public OnDiskInvertedIndexOpContext(BTree btree) throws HyracksDataException {
// TODO: Ignore opcallbacks for now.
@@ -96,4 +98,11 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
btreeCursor.destroy();
}
}
+
+ public LSMInvertedIndexSearchCursorInitialState getCursorInitialState() {
+ if (cursorInitialState == null) {
+ cursorInitialState = new LSMInvertedIndexSearchCursorInitialState();
+ }
+ return cursorInitialState;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index d9e7d34..9d99c9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -26,7 +26,6 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.InvertedListCursor;
import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
@@ -41,7 +40,7 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
private final BTree btree;
private final IIndexAccessor btreeAccessor;
- private final IInPlaceInvertedIndex invIndex;
+ private final OnDiskInvertedIndex invIndex;
private final IIndexOperationContext opCtx;
private final InvertedListCursor invListRangeSearchCursor;
private boolean isInvListCursorOpen;
@@ -50,11 +49,11 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
private RangePredicate btreePred;
private final PermutingTupleReference tokenTuple;
- private ConcatenatingTupleReference concatTuple;
+ private final ConcatenatingTupleReference concatTuple;
- public OnDiskInvertedIndexRangeSearchCursor(IInPlaceInvertedIndex invIndex, IIndexOperationContext opCtx)
+ public OnDiskInvertedIndexRangeSearchCursor(OnDiskInvertedIndex invIndex, IIndexOperationContext opCtx)
throws HyracksDataException {
- this.btree = ((OnDiskInvertedIndex) invIndex).getBTree();
+ this.btree = invIndex.getBTree();
this.btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
this.invIndex = invIndex;
this.opCtx = opCtx;
@@ -87,11 +86,7 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
return true;
}
// The current inverted-list-range-search cursor is exhausted.
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
+ invListRangeSearchCursor.close();
isInvListCursorOpen = false;
openInvListRangeSearchCursor();
return isInvListCursorOpen;
@@ -109,14 +104,8 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
@Override
public void doDestroy() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- isInvListCursorOpen = false;
- invListRangeSearchCursor.destroy();
- }
- }
+ invListRangeSearchCursor.destroy();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.destroy();
}
@@ -125,14 +114,8 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
@Override
public void doClose() throws HyracksDataException {
try {
- if (isInvListCursorOpen) {
- try {
- invListRangeSearchCursor.unloadPages();
- } finally {
- invListRangeSearchCursor.close();
- }
- isInvListCursorOpen = false;
- }
+ invListRangeSearchCursor.close();
+ isInvListCursorOpen = false;
} finally {
btreeCursor.close();
}
@@ -148,7 +131,8 @@ public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
if (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
- invIndex.openInvertedListCursor(invListRangeSearchCursor, tokenTuple, opCtx);
+ invIndex.openInvertedListCursor(btreeCursor.getTuple(), invListRangeSearchCursor,
+ (OnDiskInvertedIndexOpContext) opCtx);
invListRangeSearchCursor.prepareLoadPages();
invListRangeSearchCursor.loadPages();
concatTuple.reset();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index eff4f5a..8c6b386 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -117,7 +117,7 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen
short numTokens = ShortPointable.getShort(btreeTuple.getFieldData(PARTITIONING_NUM_TOKENS_FIELD),
btreeTuple.getFieldStart(PARTITIONING_NUM_TOKENS_FIELD));
InvertedListCursor invListCursor = partSearcher.getCachedInvertedListCursor();
- openInvertedListCursor(btreeTuple, invListCursor);
+ openInvertedListCursor(btreeTuple, invListCursor, ctx);
invListPartitions.addInvertedListCursor(invListCursor, numTokens);
tokenExists = true;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
index ba54ed4..1e812b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/InvertedIndexUtils.java
@@ -62,6 +62,7 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskI
import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.util.trace.ITracer;
public class InvertedIndexUtils {
@@ -129,7 +130,7 @@ public class InvertedIndexUtils {
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ IMetadataPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -166,7 +167,7 @@ public class InvertedIndexUtils {
filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager, invListTypeTraits,
invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker,
ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, durable);
+ invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager,
@@ -178,7 +179,7 @@ public class InvertedIndexUtils {
ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable,
- IPageManagerFactory pageManagerFactory) throws HyracksDataException {
+ IPageManagerFactory pageManagerFactory, ITracer tracer) throws HyracksDataException {
BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, invListTypeTraits,
invListCmpFactories, diskBufferCache, pageManagerFactory);
@@ -215,6 +216,6 @@ public class InvertedIndexUtils {
filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskBufferCache, fileManager,
invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterFields,
- filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
+ filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, tracer);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/8aa13cac/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index ae8713c..2435cc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -55,6 +55,9 @@ import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokeniz
import org.apache.hyracks.storage.am.lsm.invertedindex.util.LSMInvertedIndexTestUtils.HyracksTaskTestContext;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexAccessParameters;
+import org.apache.hyracks.util.trace.ITraceCategoryRegistry;
+import org.apache.hyracks.util.trace.TraceCategoryRegistry;
+import org.apache.hyracks.util.trace.Tracer;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
@@ -184,7 +187,9 @@ public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(),
+ new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()));
break;
}
case PARTITIONED_LSM: {
@@ -195,7 +200,9 @@ public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
harness.getOperationTracker(), harness.getIOScheduler(),
harness.getIOOperationCallbackFactory(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory());
+ invertedIndexFieldsForNonBulkLoadOps, true, harness.getMetadataPageManagerFactory(),
+ new Tracer(LSMInvertedIndexTestContext.class.getSimpleName(),
+ ITraceCategoryRegistry.CATEGORIES_ALL, new TraceCategoryRegistry()));
break;
}
default: {