You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2018/02/13 00:09:53 UTC
[4/7] asterixdb git commit: [ASTERIXDB-2204][STO] Fix implementations
and usages of IIndexCursor
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
index 7bf5322..b2660a4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/inmemory/InMemoryInvertedListCursor.java
@@ -81,7 +81,11 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
@Override
public int compareTo(IInvertedListCursor cursor) {
- return size() - cursor.size();
+ try {
+ return size() - cursor.size();
+ } catch (HyracksDataException hde) {
+ throw new IllegalStateException(hde);
+ }
}
public void reset(ITupleReference tuple) throws HyracksDataException {
@@ -113,7 +117,7 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
@Override
public void unpinPages() throws HyracksDataException {
if (cursorNeedsClose) {
- btreeCursor.destroy();
+ btreeCursor.close();
cursorNeedsClose = false;
}
}
@@ -135,29 +139,22 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
}
@Override
- public int size() {
+ public int size() throws HyracksDataException {
if (numElements < 0) {
btreePred.setLowKeyComparator(tokenFieldsCmp);
btreePred.setHighKeyComparator(tokenFieldsCmp);
btreePred.setLowKey(tokenTuple, true);
btreePred.setHighKey(tokenTuple, true);
-
// Perform the count.
+ btreeAccessor.search(countingCursor, btreePred);
try {
- btreeAccessor.search(countingCursor, btreePred);
while (countingCursor.hasNext()) {
countingCursor.next();
ITupleReference countTuple = countingCursor.getTuple();
numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
}
- } catch (HyracksDataException e) {
- e.printStackTrace();
} finally {
- try {
- countingCursor.destroy();
- } catch (HyracksDataException e) {
- e.printStackTrace();
- }
+ countingCursor.close();
}
}
return numElements;
@@ -197,7 +194,6 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
try {
containsKey = btreeCursor.hasNext();
} finally {
- btreeCursor.destroy();
btreeCursor.close();
btreeSearchTuple.removeLastTuple();
}
@@ -219,7 +215,6 @@ public class InMemoryInvertedListCursor implements IInvertedListCursor {
strBuilder.append(o.toString() + " ");
}
} finally {
- btreeCursor.destroy();
btreeCursor.close();
}
btreeAccessor.search(btreeCursor, btreePred);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
index 75bdbc4..eec2993 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeElementInvertedListCursor.java
@@ -70,10 +70,7 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
@Override
public boolean hasNext() {
- if (currentElementIx < numElements)
- return true;
- else
- return false;
+ return currentElementIx < numElements;
}
@Override
@@ -203,8 +200,9 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
DataInput dataIn = new DataInputStream(inStream);
Object o = serdes[i].deserialize(dataIn);
strBuilder.append(o.toString());
- if (i + 1 < tuple.getFieldCount())
+ if (i + 1 < tuple.getFieldCount()) {
strBuilder.append(",");
+ }
}
strBuilder.append(" ");
}
@@ -217,6 +215,7 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
return strBuilder.toString();
}
+ @Override
@SuppressWarnings("rawtypes")
public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
StringBuilder strBuilder = new StringBuilder();
@@ -226,8 +225,9 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
DataInput dataIn = new DataInputStream(inStream);
Object o = serdes[i].deserialize(dataIn);
strBuilder.append(o.toString());
- if (i + 1 < tuple.getFieldCount())
+ if (i + 1 < tuple.getFieldCount()) {
strBuilder.append(",");
+ }
}
return strBuilder.toString();
}
@@ -249,17 +249,23 @@ public class FixedSizeElementInvertedListCursor implements IInvertedListCursor {
}
}
- if (begin > arr.length - 1)
+ if (begin > arr.length - 1) {
return -1;
- if (key < arr[begin])
+ }
+ if (key < arr[begin]) {
return begin;
- else
+ } else {
return -1;
+ }
}
@Override
public int compareTo(IInvertedListCursor invListCursor) {
- return numElements - invListCursor.size();
+ try {
+ return numElements - invListCursor.size();
+ } catch (HyracksDataException hde) {
+ throw new IllegalStateException(hde);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
index 4942eda..8e8cb13 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java
@@ -206,7 +206,6 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
listCursor.reset(0, 0, 0, 0);
}
} finally {
- ctx.getBtreeCursor().destroy();
ctx.getBtreeCursor().close();
}
}
@@ -420,6 +419,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
private final OnDiskInvertedIndex index;
private final IInvertedIndexSearcher searcher;
private final IIndexOperationContext opCtx = new OnDiskInvertedIndexOpContext(btree);
+ private boolean destroyed = false;
public OnDiskInvertedIndexAccessor(OnDiskInvertedIndex index) throws HyracksDataException {
this.index = index;
@@ -483,6 +483,15 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
public void upsert(ITupleReference tuple) throws HyracksDataException {
throw new UnsupportedOperationException("Upsert not supported by inverted index.");
}
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ opCtx.destroy();
+ }
}
@Override
@@ -542,24 +551,38 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
btree.validate();
// Scan the btree and validate the order of elements in each inverted-list.
IIndexAccessor btreeAccessor = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
- IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
- MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
- RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
+ try {
+ MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
+ RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
+ IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
+ try {
+ btreeAccessor.search(btreeCursor, rangePred);
+ try {
+ doValidate(btreeCursor);
+ } finally {
+ btreeCursor.close();
+ }
+ } finally {
+ btreeCursor.destroy();
+ }
+ } finally {
+ btreeAccessor.destroy();
+ }
+ }
+
+ private void doValidate(IIndexCursor btreeCursor) throws HyracksDataException {
int[] fieldPermutation = new int[tokenTypeTraits.length];
for (int i = 0; i < tokenTypeTraits.length; i++) {
fieldPermutation[i] = i;
}
PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
-
+ // Search key for finding an inverted-list in the actual index.
+ ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
+ ArrayTupleReference prevTuple = new ArrayTupleReference();
IInvertedIndexAccessor invIndexAccessor = createAccessor(NoOpIndexAccessParameters.INSTANCE);
- IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
- MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
-
try {
- // Search key for finding an inverted-list in the actual index.
- ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
- ArrayTupleReference prevTuple = new ArrayTupleReference();
- btreeAccessor.search(btreeCursor, rangePred);
+ IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
+ MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
while (btreeCursor.hasNext()) {
btreeCursor.next();
tokenTuple.reset(btreeCursor.getTuple());
@@ -578,9 +601,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
invListCursor.next();
ITupleReference invListElement = invListCursor.getTuple();
// Compare with previous element.
- if (invListCmp.compare(invListElement, prevTuple) <= 0) {
- throw new HyracksDataException("Index validation failed.");
- }
+ validateWithPrevious(invListCmp, invListElement, prevTuple);
// Set new prevTuple.
TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
@@ -590,7 +611,14 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex {
}
}
} finally {
- btreeCursor.destroy();
+ invIndexAccessor.destroy();
+ }
+ }
+
+ private void validateWithPrevious(MultiComparator invListCmp, ITupleReference invListElement,
+ ArrayTupleReference prevTuple) throws HyracksDataException {
+ if (invListCmp.compare(invListElement, prevTuple) <= 0) {
+ throw new HyracksDataException("Index validation failed.");
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
index 623fc74..89d4e9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexOpContext.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
@@ -36,6 +37,7 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
private MultiComparator searchCmp;
// For prefix search on partitioned indexes.
private MultiComparator prefixSearchCmp;
+ private boolean destroyed = false;
public OnDiskInvertedIndexOpContext(BTree btree) {
// TODO: Ignore opcallbacks for now.
@@ -82,4 +84,16 @@ public class OnDiskInvertedIndexOpContext implements IIndexOperationContext {
return prefixSearchCmp;
}
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ try {
+ btreeAccessor.destroy();
+ } finally {
+ btreeCursor.destroy();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
index d1e6531..7af35ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexRangeSearchCursor.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.tuples.ConcatenatingTupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
@@ -36,7 +37,7 @@ import org.apache.hyracks.storage.common.ISearchPredicate;
/**
* Scans a range of tokens, returning tuples containing a token and an inverted-list element.
*/
-public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
+public class OnDiskInvertedIndexRangeSearchCursor extends EnforcedIndexCursor {
private final BTree btree;
private final IIndexAccessor btreeAccessor;
@@ -69,7 +70,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
this.btreePred = (RangePredicate) searchPred;
btreeAccessor.search(btreeCursor, btreePred);
invListCursor.pinPages();
@@ -77,7 +78,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
if (invListCursor.hasNext()) {
return true;
}
@@ -100,7 +101,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
invListCursor.next();
if (concatTuple.hasMaxTuples()) {
concatTuple.removeLastTuple();
@@ -109,7 +110,7 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
}
@Override
- public void destroy() throws HyracksDataException {
+ public void doDestroy() throws HyracksDataException {
if (unpinNeeded) {
invListCursor.unpinPages();
unpinNeeded = false;
@@ -118,16 +119,16 @@ public class OnDiskInvertedIndexRangeSearchCursor implements IIndexCursor {
}
@Override
- public void close() throws HyracksDataException {
+ public void doClose() throws HyracksDataException {
if (unpinNeeded) {
invListCursor.unpinPages();
unpinNeeded = false;
}
- btreeCursor.destroy();
+ btreeCursor.close();
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
return concatTuple;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
index 97f1f76..0563ec9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexSearchCursor.java
@@ -27,11 +27,11 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
-import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
-public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
+public class OnDiskInvertedIndexSearchCursor extends EnforcedIndexCursor {
private List<ByteBuffer> resultBuffers;
private int numResultBuffers;
@@ -55,7 +55,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
currentBufferIndex = 0;
tupleIndex = 0;
resultBuffers = invIndexSearcher.getResultBuffers();
@@ -66,7 +66,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
}
@Override
- public boolean hasNext() {
+ public boolean doHasNext() {
if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount()) {
return true;
} else {
@@ -75,7 +75,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
}
@Override
- public void next() {
+ public void doNext() {
frameTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
resultTuple.reset(frameTuple);
tupleIndex++;
@@ -89,12 +89,12 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
return resultTuple;
}
@Override
- public void close() {
+ public void doClose() {
currentBufferIndex = 0;
tupleIndex = 0;
invIndexSearcher.reset();
@@ -103,7 +103,7 @@ public class OnDiskInvertedIndexSearchCursor implements IIndexCursor {
}
@Override
- public void destroy() throws HyracksDataException {
+ public void doDestroy() throws HyracksDataException {
currentBufferIndex = 0;
tupleIndex = 0;
resultBuffers = null;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
index 01e0684..064a26d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndex.java
@@ -87,8 +87,8 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen
}
ctx.getBtreePred().setLowKey(lowSearchKey, true);
ctx.getBtreePred().setHighKey(highSearchKey, true);
- ctx.getBtreeAccessor().search(ctx.getBtreeCursor(), ctx.getBtreePred());
boolean tokenExists = false;
+ ctx.getBtreeAccessor().search(ctx.getBtreeCursor(), ctx.getBtreePred());
try {
while (ctx.getBtreeCursor().hasNext()) {
ctx.getBtreeCursor().next();
@@ -102,7 +102,6 @@ public class PartitionedOnDiskInvertedIndex extends OnDiskInvertedIndex implemen
tokenExists = true;
}
} finally {
- ctx.getBtreeCursor().destroy();
ctx.getBtreeCursor().close();
}
return tokenExists;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
index ff5306c..5ae4e05 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java
@@ -92,16 +92,14 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
IPartitionedInvertedIndex partInvIndex = (IPartitionedInvertedIndex) invIndex;
searchResult.reset();
if (partInvIndex.isEmpty()) {
+ resultCursor.open(null, searchPred);
return;
}
-
tokenizeQuery(searchPred);
short numQueryTokens = (short) queryTokenAppender.getTupleCount();
-
IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier();
short numTokensLowerBound = searchModifier.getNumTokensLowerBound(numQueryTokens);
short numTokensUpperBound = searchModifier.getNumTokensUpperBound(numQueryTokens);
-
occurrenceThreshold = searchModifier.getOccurrenceThreshold(numQueryTokens);
if (occurrenceThreshold <= 0) {
throw HyracksDataException.create(ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION);
@@ -118,15 +116,14 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
maxCountPossible--;
// No results possible.
if (maxCountPossible < occurrenceThreshold) {
+ resultCursor.open(null, searchPred);
return;
}
}
}
-
ArrayList<IInvertedListCursor>[] partitionCursors = partitions.getPartitions();
short start = partitions.getMinValidPartitionIndex();
short end = partitions.getMaxValidPartitionIndex();
-
// Typically, we only enter this case for disk-based inverted indexes.
// TODO: This behavior could potentially lead to a deadlock if we cannot pin
// all inverted lists in memory, and are forced to wait for a page to get evicted
@@ -165,7 +162,6 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher
invListMerger.reset();
invListMerger.merge(partitionCursors[i], occurrenceThreshold, numPrefixLists, searchResult);
}
-
resultCursor.open(null, searchPred);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
index 856bf6b..1c118de 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
@@ -16,17 +16,15 @@
! specific language governing permissions and limitations
! under the License.
!-->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>hyracks-storage-am-lsm-rtree</artifactId>
-
<parent>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks</artifactId>
<version>0.3.4-SNAPSHOT</version>
</parent>
-
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -35,11 +33,9 @@
<comments>A business-friendly OSS license</comments>
</license>
</licenses>
-
<properties>
<root.dir>${basedir}/../..</root.dir>
</properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
@@ -91,5 +87,9 @@
<artifactId>hyracks-storage-am-bloomfilter</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
index 09237a3..9be61fc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -68,4 +69,9 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
public LSMRTreeCursorInitialState getInitialState() {
return initialState;
}
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ // No Op
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 7172b74..d891d9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -22,6 +22,8 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -32,7 +34,6 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
@@ -115,124 +116,247 @@ public class LSMRTree extends AbstractLSMRTree {
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
-
// scan the memory RTree
- RTreeAccessor memRTreeAccessor =
- flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
- RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false);
- SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
- memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
-
- ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(),
- flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
-
- //count the number of tuples in the buddy btree
+ TreeTupleSorter rTreeTupleSorter = null;
+ MutableBoolean isEmpty = new MutableBoolean(true);
+ MutableLong numBTreeTuples = new MutableLong(0L);
+ ILSMDiskComponent component;
+ ILSMDiskComponentBulkLoader componentBulkLoader = null;
+ RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
BTreeAccessor memBTreeAccessor =
flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
- RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
- IIndexCursor btreeCountingCursor = memBTreeAccessor.createCountingSearchCursor();
- memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate);
- long numBTreeTuples = 0L;
+ boolean abort = true;
try {
- while (btreeCountingCursor.hasNext()) {
- btreeCountingCursor.next();
- ITupleReference countTuple = btreeCountingCursor.getTuple();
- numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
+ try {
+ rTreeTupleSorter = getRTreeTupleSorter(flushingComponent, memBTreeAccessor, btreeNullPredicate,
+ numBTreeTuples, isEmpty);
+ rTreeTupleSorter.sort();
+ component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(),
+ flushOp.getBloomFilterTarget(), true);
+ componentBulkLoader =
+ component.createBulkLoader(1.0f, false, numBTreeTuples.longValue(), false, false, false);
+ flushLoadRTree(isEmpty, rTreeTupleSorter, componentBulkLoader);
+ // scan the memory BTree and bulk load delete tuples
+ flushLoadBtree(memBTreeAccessor, componentBulkLoader, btreeNullPredicate);
+ } finally {
+ try {
+ memBTreeAccessor.destroy();
+ } finally {
+ if (rTreeTupleSorter != null) {
+ rTreeTupleSorter.destroy();
+ }
+ }
}
- } finally {
- btreeCountingCursor.destroy();
- }
-
- ILSMDiskComponentBulkLoader componentBulkLoader =
- component.createBulkLoader(1.0f, false, numBTreeTuples, false, false, false);
-
- ITreeIndexCursor cursor;
- IBinaryComparatorFactory[] linearizerArray = { linearizer };
-
- TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(),
- linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
- flushingComponent.getIndex().getBufferCache(), comparatorFields);
-
- // BulkLoad the tuples from the in-memory tree into the new disk
- // RTree.
- boolean isEmpty = true;
- try {
- while (rtreeScanCursor.hasNext()) {
- isEmpty = false;
- rtreeScanCursor.next();
- rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
}
+ // Note. If we change the filter to write to metadata object, we don't need the if block above
+ flushingComponent.getMetadata().copy(component.getMetadata());
+ abort = false;
+ componentBulkLoader.end();
} finally {
- rtreeScanCursor.destroy();
+ if (abort && componentBulkLoader != null) {
+ componentBulkLoader.abort();
+ }
}
- rTreeTupleSorter.sort();
-
- cursor = rTreeTupleSorter;
+ return component;
+ }
- if (!isEmpty) {
+ private void flushLoadRTree(MutableBoolean isEmpty, TreeTupleSorter rTreeTupleSorter,
+ ILSMDiskComponentBulkLoader componentBulkLoader) throws HyracksDataException {
+ if (!isEmpty.booleanValue()) {
+ rTreeTupleSorter.open(null, null);
try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference frameTuple = cursor.getTuple();
+ while (rTreeTupleSorter.hasNext()) {
+ rTreeTupleSorter.next();
+ ITupleReference frameTuple = rTreeTupleSorter.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
- cursor.destroy();
+ rTreeTupleSorter.close();
}
}
+ }
- // scan the memory BTree
+ private void flushLoadBtree(BTreeAccessor memBTreeAccessor, ILSMDiskComponentBulkLoader componentBulkLoader,
+ RangePredicate btreeNullPredicate) throws HyracksDataException {
IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
- memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
try {
- while (btreeScanCursor.hasNext()) {
- btreeScanCursor.next();
- ITupleReference frameTuple = btreeScanCursor.getTuple();
- componentBulkLoader.delete(frameTuple);
+ memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
+ try {
+ while (btreeScanCursor.hasNext()) {
+ btreeScanCursor.next();
+ ITupleReference frameTuple = btreeScanCursor.getTuple();
+ componentBulkLoader.delete(frameTuple);
+ }
+ } finally {
+ btreeScanCursor.close();
}
} finally {
btreeScanCursor.destroy();
}
+ }
- if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
- filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
- filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
- getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
+ private TreeTupleSorter getRTreeTupleSorter(LSMRTreeMemoryComponent flushingComponent,
+ BTreeAccessor memBTreeAccessor, RangePredicate btreeNullPredicate, MutableLong numBTreeTuples,
+ MutableBoolean isEmpty) throws HyracksDataException {
+ RTreeAccessor memRTreeAccessor =
+ flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ try {
+ RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false);
+ try {
+ SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
+ memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
+ try {
+ //count the number of tuples in the buddy btree
+ countTuples(memBTreeAccessor, btreeNullPredicate, numBTreeTuples);
+ IBinaryComparatorFactory[] linearizerArray = { linearizer };
+ boolean failed = true;
+ TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(),
+ linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
+ flushingComponent.getIndex().getBufferCache(), comparatorFields);
+ try {
+ // BulkLoad the tuples from the in-memory tree into the new disk
+ // RTree.
+ isEmpty.setValue(fill(rtreeScanCursor, rTreeTupleSorter));
+ failed = false;
+ } finally {
+ if (failed) {
+ rTreeTupleSorter.destroy();
+ }
+ }
+ return rTreeTupleSorter;
+ } finally {
+ rtreeScanCursor.close();
+ }
+ } finally {
+ rtreeScanCursor.destroy();
+ }
+ } finally {
+ memRTreeAccessor.destroy();
}
- // Note. If we change the filter to write to metadata object, we don't need the if block above
- flushingComponent.getMetadata().copy(component.getMetadata());
+ }
- componentBulkLoader.end();
- return component;
+ private boolean fill(RTreeSearchCursor rtreeScanCursor, TreeTupleSorter rTreeTupleSorter)
+ throws HyracksDataException {
+ boolean isEmpty = true;
+ while (rtreeScanCursor.hasNext()) {
+ isEmpty = false;
+ rtreeScanCursor.next();
+ rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
+ }
+ return isEmpty;
+ }
+
+ private void countTuples(BTreeAccessor memBTreeAccessor, RangePredicate btreeNullPredicate,
+ MutableLong numBTreeTuples) throws HyracksDataException {
+ IIndexCursor btreeCountingCursor = memBTreeAccessor.createCountingSearchCursor();
+ try {
+ memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate);
+ try {
+ while (btreeCountingCursor.hasNext()) {
+ btreeCountingCursor.next();
+ ITupleReference countTuple = btreeCountingCursor.getTuple();
+ numBTreeTuples.setValue(
+ IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0)));
+ }
+ } finally {
+ btreeCountingCursor.close();
+ }
+ } finally {
+ btreeCountingCursor.destroy();
+ }
}
@Override
public ILSMDiskComponent doMerge(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
+ ILSMDiskComponentBulkLoader componentBulkLoader = null;
+ ILSMDiskComponent mergedComponent = null;
+ boolean abort = true;
+ try {
+ mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(),
+ mergeOp.getBloomFilterTarget(), true);
+ componentBulkLoader = loadMergeBulkLoader(mergeOp, cursor, mergedComponent);
+ if (mergedComponent.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
+ filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+ }
+ getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(),
+ mergedComponent.getMetadataHolder());
+ }
+ abort = false;
+ componentBulkLoader.end();
+ } finally {
+ try {
+ cursor.destroy();
+ } finally {
+ if (abort && componentBulkLoader != null) {
+ componentBulkLoader.abort();
+ }
+ }
+ }
+ return mergedComponent;
+ }
+
+ private ILSMDiskComponentBulkLoader loadMergeBulkLoader(LSMRTreeMergeOperation mergeOp, IIndexCursor cursor,
+ ILSMDiskComponent mergedComponent) throws HyracksDataException {
+ ILSMDiskComponentBulkLoader componentBulkLoader = null;
+ boolean abort = true;
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
search(opCtx, cursor, rtreeSearchPred);
-
- ILSMDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
- mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- ILSMDiskComponentBulkLoader componentBulkLoader;
- // In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that
- // lsmHarness.endSearch() is called once when the r-trees have been merged.
- if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- // Keep the deleted tuples since the oldest disk component is not included in the merge operation
-
- long numElements = 0L;
- for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
- .getNumElements();
+ try {
+ try {
+ // In case we must keep the deleted-keys BTrees, then they must be merged
+ // *before* merging the r-trees so that
+ // lsmHarness.endSearch() is called once when the r-trees have been merged.
+ if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents
+ .get(diskComponents.size() - 1)) {
+ // Keep the deleted tuples since the oldest disk component
+ // is not included in the merge operation
+ long numElements = 0L;
+ for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
+ numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
+ .getNumElements();
+ }
+ componentBulkLoader =
+ mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false);
+ mergeLoadBTree(opCtx, rtreeSearchPred, componentBulkLoader);
+ } else {
+ //no buddy-btree needed
+ componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false);
+ }
+ //search old rtree components
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference frameTuple = cursor.getTuple();
+ componentBulkLoader.add(frameTuple);
+ }
+ } finally {
+ cursor.close();
+ }
+ abort = false;
+ } finally {
+ if (abort && componentBulkLoader != null) {
+ componentBulkLoader.abort();
}
- componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false);
+ }
+ return componentBulkLoader;
+ }
- LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
+ private void mergeLoadBTree(ILSMIndexOperationContext opCtx, ISearchPredicate rtreeSearchPred,
+ ILSMDiskComponentBulkLoader componentBulkLoader) throws HyracksDataException {
+ LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
+ try {
search(opCtx, btreeCursor, rtreeSearchPred);
try {
while (btreeCursor.hasNext()) {
@@ -241,38 +365,11 @@ public class LSMRTree extends AbstractLSMRTree {
componentBulkLoader.delete(tuple);
}
} finally {
- btreeCursor.destroy();
- }
- } else {
- //no buddy-btree needed
- componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false);
- }
-
- //search old rtree components
- try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference frameTuple = cursor.getTuple();
- componentBulkLoader.add(frameTuple);
+ btreeCursor.close();
}
} finally {
- cursor.destroy();
+ btreeCursor.destroy();
}
-
- if (mergedComponent.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
- for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
- filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
- }
- getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
- getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(),
- mergedComponent.getMetadataHolder());
- }
-
- componentBulkLoader.end();
-
- return mergedComponent;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index e4267e2..d41e406 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -41,12 +41,12 @@ import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTree.RTreeAccessor;
import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
-import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
-public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor {
+public abstract class LSMRTreeAbstractCursor extends EnforcedIndexCursor implements ILSMIndexCursor {
protected boolean open;
protected RTreeSearchCursor[] rtreeCursors;
@@ -76,7 +76,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor {
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
if (btreeCmp == null) {
btreeCmp = lsmInitialState.getBTreeCmp();
@@ -153,7 +153,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor {
}
@Override
- public void destroy() throws HyracksDataException {
+ public void doDestroy() throws HyracksDataException {
if (!open) {
return;
}
@@ -176,7 +176,7 @@ public abstract class LSMRTreeAbstractCursor implements ILSMIndexCursor {
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
return frameTuple;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java
index df4e16e..892fe83 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDeletedKeysBTreeMergeCursor.java
@@ -46,7 +46,7 @@ public class LSMRTreeDeletedKeysBTreeMergeCursor extends LSMIndexSearchCursor {
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
cmp = lsmInitialState.getBTreeCmp();
operationalComponents = lsmInitialState.getOperationalComponents();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 50f1961..c3c74b8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -22,6 +22,8 @@ package org.apache.hyracks.storage.am.lsm.rtree.impls;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.util.DestroyUtils;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
@@ -51,6 +53,7 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
private RTreeOpContext currentRTreeOpContext;
private BTreeOpContext currentBTreeOpContext;
private LSMRTreeCursorInitialState searchInitialState;
+ private boolean destroyed = false;
public LSMRTreeOpContext(ILSMIndex index, List<ILSMMemoryComponent> mutableComponents,
ITreeIndexFrameFactory rtreeLeafFrameFactory, ITreeIndexFrameFactory rtreeInteriorFrameFactory,
@@ -128,4 +131,19 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
public RTreeOpContext getCurrentRTreeOpContext() {
return currentRTreeOpContext;
}
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ if (destroyed) {
+ return;
+ }
+ destroyed = true;
+ Throwable failure = DestroyUtils.destroy(null, mutableRTreeAccessors);
+ failure = DestroyUtils.destroy(failure, rtreeOpContexts);
+ failure = DestroyUtils.destroy(failure, mutableBTreeAccessors);
+ failure = DestroyUtils.destroy(failure, btreeOpContexts);
+ if (failure != null) {
+ throw HyracksDataException.create(failure);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
index 0edf22f..d485f64 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSearchCursor.java
@@ -39,13 +39,13 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
}
@Override
- public void destroy() throws HyracksDataException {
- super.destroy();
+ public void doDestroy() throws HyracksDataException {
+ super.doDestroy();
currentCursor = 0;
}
@Override
- public void close() throws HyracksDataException {
+ public void doClose() throws HyracksDataException {
if (!open) {
return;
}
@@ -54,8 +54,8 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
foundNext = false;
try {
for (int i = 0; i < numberOfTrees; i++) {
- rtreeCursors[i].destroy();
- btreeCursors[i].destroy();
+ rtreeCursors[i].close();
+ btreeCursors[i].close();
}
rtreeCursors = null;
btreeCursors = null;
@@ -88,7 +88,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
if (foundNext) {
return true;
}
@@ -111,7 +111,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
killerTupleFound = true;
}
} finally {
- btreeCursors[i].destroy();
+ btreeCursors[i].close();
}
}
if (!killerTupleFound) {
@@ -120,7 +120,7 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
return true;
}
}
- rtreeCursors[currentCursor].destroy();
+ rtreeCursors[currentCursor].close();
currentCursor++;
searchNextCursor();
}
@@ -128,13 +128,13 @@ public class LSMRTreeSearchCursor extends LSMRTreeAbstractCursor {
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
foundNext = false;
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
- super.open(initialState, searchPred);
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ super.doOpen(initialState, searchPred);
searchNextCursor();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
index d66e882..9bbc3e1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeSortedCursor.java
@@ -51,17 +51,21 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor {
}
@Override
- public void close() throws HyracksDataException {
+ public void doClose() throws HyracksDataException {
depletedRtreeCursors = new boolean[numberOfTrees];
foundNext = false;
try {
for (int i = 0; i < numberOfTrees; i++) {
rtreeCursors[i].close();
rtreeAccessors[i].search(rtreeCursors[i], rtreeSearchPredicate);
- if (rtreeCursors[i].hasNext()) {
- rtreeCursors[i].next();
- } else {
- depletedRtreeCursors[i] = true;
+ try {
+ if (rtreeCursors[i].hasNext()) {
+ rtreeCursors[i].next();
+ } else {
+ depletedRtreeCursors[i] = true;
+ }
+ } finally {
+ rtreeCursors[i].close();
}
}
} finally {
@@ -88,7 +92,7 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor {
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
while (!foundNext) {
frameTuple = null;
@@ -138,7 +142,7 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor {
break;
}
} finally {
- btreeCursors[i].destroy();
+ btreeCursors[i].close();
}
}
if (!killed) {
@@ -150,14 +154,13 @@ public class LSMRTreeSortedCursor extends LSMRTreeAbstractCursor {
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
foundNext = false;
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
- super.open(initialState, searchPred);
-
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ super.doOpen(initialState, searchPred);
depletedRtreeCursors = new boolean[numberOfTrees];
foundNext = false;
for (int i = 0; i < numberOfTrees; i++) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 94d76f6..cade80f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -92,85 +92,133 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
- RTreeAccessor memRTreeAccessor =
- flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
- RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
- memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- ILSMDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
- ILSMDiskComponentBulkLoader componentBulkLoader =
- component.createBulkLoader(1.0f, false, 0L, false, false, false);
-
- // Since the LSM-RTree is used as a secondary assumption, the
- // primary key will be the last comparator in the BTree comparators
- TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(),
- linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
- flushingComponent.getIndex().getBufferCache(), comparatorFields);
-
+ ILSMDiskComponent component = null;
+ ILSMDiskComponentBulkLoader componentBulkLoader = null;
+ TreeTupleSorter rTreeTupleSorter = null;
+ TreeTupleSorter bTreeTupleSorter = null;
boolean isEmpty = true;
+ boolean abort = true;
try {
- while (rtreeScanCursor.hasNext()) {
- isEmpty = false;
- rtreeScanCursor.next();
- rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
- }
- } finally {
- rtreeScanCursor.destroy();
- }
- if (!isEmpty) {
- rTreeTupleSorter.sort();
- }
+ RTreeAccessor memRTreeAccessor =
+ flushingComponent.getIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
- // scan the memory BTree
- BTreeAccessor memBTreeAccessor =
- flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
- BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
- RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
- memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
- TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(),
- linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
- flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields);
-
- isEmpty = true;
- try {
- while (btreeScanCursor.hasNext()) {
- isEmpty = false;
- btreeScanCursor.next();
- bTreeTupleSorter.insertTupleEntry(btreeScanCursor.getPageId(), btreeScanCursor.getTupleOffset());
+ try {
+ RTreeSearchCursor rtreeScanCursor = memRTreeAccessor.createSearchCursor(false);
+ try {
+ memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
+ component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
+ componentBulkLoader = component.createBulkLoader(1.0f, false, 0L, false, false, false);
+ // Since the LSM-RTree is used as a secondary assumption, the
+ // primary key will be the last comparator in the BTree comparators
+ rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getIndex().getFileId(), linearizerArray,
+ rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(),
+ flushingComponent.getIndex().getBufferCache(), comparatorFields);
+ try {
+ isEmpty = scanAndSort(rtreeScanCursor, rTreeTupleSorter);
+ } finally {
+ rtreeScanCursor.close();
+ }
+ } finally {
+ rtreeScanCursor.destroy();
+ }
+ } finally {
+ memRTreeAccessor.destroy();
}
- } finally {
- btreeScanCursor.destroy();
- }
- if (!isEmpty) {
- bTreeTupleSorter.sort();
- }
-
- LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor(rTreeTupleSorter,
- bTreeTupleSorter, comparatorFields, linearizerArray);
- cursor.open(null, null);
-
- try {
- while (cursor.hasNext()) {
- cursor.next();
- ITupleReference frameTuple = cursor.getTuple();
-
- componentBulkLoader.add(frameTuple);
+ if (!isEmpty) {
+ rTreeTupleSorter.sort();
+ }
+ // scan the memory BTree
+ RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
+ BTreeAccessor memBTreeAccessor =
+ flushingComponent.getBuddyIndex().createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ try {
+ bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBuddyIndex().getFileId(), linearizerArray,
+ btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(),
+ flushingComponent.getBuddyIndex().getBufferCache(), comparatorFields);
+ BTreeRangeSearchCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
+ try {
+ isEmpty = true;
+ memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
+ try {
+ isEmpty = scanAndSort(btreeScanCursor, bTreeTupleSorter);
+ } finally {
+ btreeScanCursor.close();
+ }
+ } finally {
+ btreeScanCursor.destroy();
+ }
+ } finally {
+ memBTreeAccessor.destroy();
+ }
+ if (!isEmpty) {
+ bTreeTupleSorter.sort();
+ }
+ LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor(
+ rTreeTupleSorter, bTreeTupleSorter, comparatorFields, linearizerArray);
+ try {
+ cursor.open(null, null);
+ try {
+ while (cursor.hasNext()) {
+ cursor.next();
+ ITupleReference frameTuple = cursor.getTuple();
+ componentBulkLoader.add(frameTuple);
+ }
+ } finally {
+ cursor.close();
+ }
+ } finally {
+ cursor.destroy();
+ }
+ if (component.getLSMComponentFilter() != null) {
+ List<ITupleReference> filterTuples = new ArrayList<>();
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
+ filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
}
+ flushingComponent.getMetadata().copy(component.getMetadata());
+ abort = false;
+ componentBulkLoader.end();
} finally {
- cursor.destroy();
+ try {
+ if (rTreeTupleSorter != null) {
+ rTreeTupleSorter.destroy();
+ }
+ } finally {
+ try {
+ if (bTreeTupleSorter != null) {
+ bTreeTupleSorter.destroy();
+ }
+ } finally {
+ if (abort && componentBulkLoader != null) {
+ componentBulkLoader.abort();
+ }
+ }
+ }
}
+ return component;
+ }
- if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<>();
- filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
- filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
- getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
+ private boolean scanAndSort(RTreeSearchCursor scanCursor, TreeTupleSorter tupleSorter) throws HyracksDataException {
+ boolean isEmpty = true;
+ while (scanCursor.hasNext()) {
+ isEmpty = false;
+ scanCursor.next();
+ tupleSorter.insertTupleEntry(scanCursor.getPageId(), scanCursor.getTupleOffset());
}
- flushingComponent.getMetadata().copy(component.getMetadata());
+ return isEmpty;
+ }
- componentBulkLoader.end();
- return component;
+ private boolean scanAndSort(BTreeRangeSearchCursor scanCursor, TreeTupleSorter tupleSorter)
+ throws HyracksDataException {
+ boolean isEmpty = true;
+ while (scanCursor.hasNext()) {
+ isEmpty = false;
+ scanCursor.next();
+ tupleSorter.insertTupleEntry(scanCursor.getPageId(), scanCursor.getTupleOffset());
+ }
+ return isEmpty;
}
@Override
@@ -193,7 +241,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
componentBulkLoader.add(frameTuple);
}
} finally {
- cursor.destroy();
+ cursor.close();
}
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
index c2a2bf5..77bf58e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
@@ -23,12 +23,12 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.ILSMIndexCursor;
+import org.apache.hyracks.storage.common.EnforcedIndexCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
-import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
-public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor {
+public class LSMRTreeWithAntiMatterTuplesFlushCursor extends EnforcedIndexCursor implements ILSMIndexCursor {
private final TreeTupleSorter rTreeTupleSorter;
private final TreeTupleSorter bTreeTupleSorter;
private final int[] comparatorFields;
@@ -48,12 +48,23 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
-
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ boolean rtreeOpen = false;
+ boolean btreeOpen = false;
+ try {
+ rTreeTupleSorter.open(initialState, searchPred);
+ rtreeOpen = true;
+ bTreeTupleSorter.open(initialState, searchPred);
+ btreeOpen = true;
+ } finally {
+ if (rtreeOpen && !btreeOpen) {
+ rTreeTupleSorter.close();
+ }
+ }
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
if (foundNext) {
return true;
}
@@ -124,22 +135,31 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ILSMIndexCursor
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
foundNext = false;
}
@Override
- public void destroy() throws HyracksDataException {
+ public void doDestroy() throws HyracksDataException {
+ try {
+ rTreeTupleSorter.destroy();
+ } finally {
+ bTreeTupleSorter.destroy();
+ }
}
@Override
- public void close() throws HyracksDataException {
-
+ public void doClose() throws HyracksDataException {
+ try {
+ rTreeTupleSorter.close();
+ } finally {
+ bTreeTupleSorter.close();
+ }
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
return frameTuple;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ff6a36d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
index b8180a6..4547063 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
@@ -70,7 +70,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ public void doOpen(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
cmp = lsmInitialState.getHilbertCmp();
btreeCmp = lsmInitialState.getBTreeCmp();
@@ -136,7 +136,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
}
@Override
- public boolean hasNext() throws HyracksDataException {
+ public boolean doHasNext() throws HyracksDataException {
if (numMemoryComponents > 0) {
if (foundNext) {
return true;
@@ -158,13 +158,13 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
return true;
}
}
- mutableRTreeCursors[currentCursor].destroy();
+ mutableRTreeCursors[currentCursor].close();
currentCursor++;
searchNextCursor();
}
- while (super.hasNext()) {
- super.next();
- ITupleReference diskRTreeTuple = super.getTuple();
+ while (super.doHasNext()) {
+ super.doNext();
+ ITupleReference diskRTreeTuple = super.doGetTuple();
// TODO: at this time, we only add proceed().
// reconcile() and complete() can be added later after considering the semantics.
@@ -178,13 +178,11 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
}
}
} else {
- if (super.hasNext()) {
- super.next();
- ITupleReference diskRTreeTuple = super.getTuple();
-
+ if (super.doHasNext()) {
+ super.doNext();
+ ITupleReference diskRTreeTuple = super.doGetTuple();
// TODO: at this time, we only add proceed() part.
// reconcile() and complete() can be added later after considering the semantics.
-
// Call proceed() to do necessary operations before returning this tuple.
// Since in-memory components don't exist, we can skip searching in-memory B-Trees.
searchCallback.proceed(diskRTreeTuple);
@@ -214,17 +212,17 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
}
@Override
- public void next() throws HyracksDataException {
+ public void doNext() throws HyracksDataException {
foundNext = false;
}
@Override
- public ITupleReference getTuple() {
+ public ITupleReference doGetTuple() {
return frameTuple;
}
@Override
- public void close() throws HyracksDataException {
+ public void doClose() throws HyracksDataException {
if (!open) {
return;
}
@@ -236,11 +234,11 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
btreeCursors[i].close();
}
}
- super.close();
+ super.doClose();
}
@Override
- public void destroy() throws HyracksDataException {
+ public void doDestroy() throws HyracksDataException {
if (!open) {
return;
}
@@ -252,7 +250,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
}
currentCursor = 0;
open = false;
- super.destroy();
+ super.doDestroy();
}
@Override
@@ -272,7 +270,7 @@ public class LSMRTreeWithAntiMatterTuplesSearchCursor extends LSMIndexSearchCurs
return false;
}
} finally {
- btreeCursors[i].destroy();
+ btreeCursors[i].close();
}
}
return true;