You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/12/20 18:29:11 UTC

Change in asterixdb[master]: [ASTERIXDB-2205][STO] Maintain includeMutableComponent corre...

abdullah alamoudi has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/2243

Change subject: [ASTERIXDB-2205][STO] Maintain includeMutableComponent correctly
......................................................................

[ASTERIXDB-2205][STO] Maintain includeMutableComponent correctly

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- This change fixes ASTERIXDB-2205. The root cause
  for ASTERIXDB-2205 was that the value of
  includeMutableComponent is not maintained correctly.

Change-Id: Ic08a9372c608d6de960e1419899530aa55aa72e0
---
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
5 files changed, 43 insertions(+), 34 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/2243/1

diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
index 2f77504..7cd309a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
@@ -68,7 +68,7 @@
         cmp = lsmInitialState.getOriginalKeyComparator();
         operationalComponents = lsmInitialState.getOperationalComponents();
         lsmHarness = lsmInitialState.getLSMHarness();
-        includeMutableComponent = false;
+        setIncludeMutableComponent(false);
         int numBTrees = operationalComponents.size();
         rangeCursors = new IIndexCursor[numBTrees];
         btreeAccessors = new BTreeAccessor[numBTrees];
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
index 3e14fb9..54e9ea4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
@@ -108,7 +108,7 @@
                 if (canCallProceed) {
                     // if there are no memory components. no need to lock at all
                     // since whatever the search reads will never changes
-                    if (includeMutableComponent) {
+                    if (isIncludeMutableComponent()) {
                         if (!searchCallback.proceed(queueHead.getTuple())) {
                             // In case proceed() fails and there is an in-memory component,
                             // we can't simply use this element since there might be a change.
@@ -127,8 +127,7 @@
                                 // Re-traverses the index.
                                 reusablePred.setLowKey(copyTuple, true);
                                 btreeAccessors[0].search(rangeCursors[0], reusablePred);
-                                //------
-                                includeMutableComponent = pushIntoQueueFromCursorAndReplaceThisElement(mutableElement);
+                                pushIntoQueueFromCursorAndReplaceThisElement(mutableElement);
                                 // now that we have completed the search and we have latches over the pages,
                                 // it is safe to complete the operation.. but as per the API of the callback
                                 // we only complete if we're producing this tuple
@@ -149,7 +148,7 @@
                             } else {
                                 // There are no more elements in the memory component.. can safely skip locking for the
                                 // remaining operations
-                                includeMutableComponent = false;
+                                setIncludeMutableComponent(false);
                             }
                         }
                     }
@@ -217,19 +216,21 @@
         opCtx.getIndex().getHarness().replaceMemoryComponentsWithDiskComponents(getOpCtx(), replaceFrom);
         // redo the search on the new component
         for (int i = replaceFrom; i < switchRequest.length; i++) {
-            if (switchRequest[i] && switchedElements[i] != null) {
-                copyTuple.reset(switchComponentTupleBuilders[i].getFieldEndOffsets(),
-                        switchComponentTupleBuilders[i].getByteArray());
-                reusablePred.setLowKey(copyTuple, true);
-                rangeCursors[i].reset();
+            if (switchRequest[i]) {
                 ILSMComponent component = operationalComponents.get(i);
                 BTree btree = (BTree) component.getIndex();
                 if (i == 0 && component.getType() != LSMComponentType.MEMORY) {
-                    includeMutableComponent = false;
+                    setIncludeMutableComponent(false);
                 }
-                btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-                btreeAccessors[i].search(rangeCursors[i], reusablePred);
-                pushIntoQueueFromCursorAndReplaceThisElement(switchedElements[i]);
+                if (switchedElements[i] != null) {
+                    copyTuple.reset(switchComponentTupleBuilders[i].getFieldEndOffsets(),
+                            switchComponentTupleBuilders[i].getByteArray());
+                    reusablePred.setLowKey(copyTuple, true);
+                    rangeCursors[i].close();
+                    btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+                    btreeAccessors[i].search(rangeCursors[i], reusablePred);
+                    pushIntoQueueFromCursorAndReplaceThisElement(switchedElements[i]);
+                }
             }
             switchRequest[i] = false;
             // any failed switch makes further switches pointless
@@ -283,7 +284,7 @@
     }
 
     private void redoMemoryComponentSearchIfNeeded() throws HyracksDataException {
-        if (!includeMutableComponent) {
+        if (!isIncludeMutableComponent()) {
             return;
         }
         // if the last n records, none were from memory and there are writers inside the component,
@@ -305,7 +306,7 @@
                 // Re-traverses the index.
                 reusablePred.setLowKey(copyTuple, true);
                 btreeAccessors[0].search(rangeCursors[0], reusablePred);
-                includeMutableComponent = pushIntoQueueFromCursorAndReplaceThisElement(mutableElement);
+                pushIntoQueueFromCursorAndReplaceThisElement(mutableElement);
             }
         }
         tupleFromMemoryComponentCount = 0;
@@ -335,7 +336,7 @@
         reusablePred.setLowKeyComparator(cmp);
         reusablePred.setHighKey(predicate.getHighKey(), predicate.isHighKeyInclusive());
         reusablePred.setHighKeyComparator(predicate.getHighKeyComparator());
-        includeMutableComponent = false;
+        setIncludeMutableComponent(false);
 
         int numBTrees = operationalComponents.size();
         if (rangeCursors == null || rangeCursors.length != numBTrees) {
@@ -354,13 +355,11 @@
                 // re-use
                 rangeCursors[i].reset();
             }
-            if (component.getType() == LSMComponentType.MEMORY) {
-                includeMutableComponent = true;
-                btree = (BTree) component.getIndex();
-            } else {
-                btree = (BTree) component.getIndex();
-            }
 
+            if (component.getType() == LSMComponentType.MEMORY) {
+                setIncludeMutableComponent(true);
+            }
+            btree = (BTree) component.getIndex();
             if (btreeAccessors[i] == null) {
                 btreeAccessors[i] = btree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             } else {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index 17c681c..3d9e216 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -50,7 +50,7 @@
     protected PriorityQueueComparator pqCmp;
     protected MultiComparator cmp;
     protected boolean needPushElementIntoQueue;
-    protected boolean includeMutableComponent;
+    private boolean includeMutableComponent;
     protected ILSMHarness lsmHarness;
     protected boolean switchPossible = true;
     protected int hasNextCallCount = 0;
@@ -194,16 +194,18 @@
         return filter == null ? null : filter.getMaxTuple();
     }
 
-    protected boolean pushIntoQueueFromCursorAndReplaceThisElement(PriorityQueueElement e) throws HyracksDataException {
+    protected void pushIntoQueueFromCursorAndReplaceThisElement(PriorityQueueElement e) throws HyracksDataException {
         int cursorIndex = e.getCursorIndex();
         if (rangeCursors[cursorIndex].hasNext()) {
             rangeCursors[cursorIndex].next();
             e.reset(rangeCursors[cursorIndex].getTuple());
             outputPriorityQueue.offer(e);
-            return true;
+            return;
         }
         rangeCursors[cursorIndex].close();
-        return false;
+        if (cursorIndex == 0) {
+            setIncludeMutableComponent(false);
+        }
     }
 
     protected boolean isDeleted(PriorityQueueElement checkElement) throws HyracksDataException {
@@ -325,4 +327,12 @@
         return cmp.compare(tupleA, tupleB);
     }
 
+    protected boolean isIncludeMutableComponent() {
+        return includeMutableComponent;
+    }
+
+    protected void setIncludeMutableComponent(boolean includeMutableComponent) {
+        this.includeMutableComponent = includeMutableComponent;
+    }
+
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
index d565b9a..b30f0f5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexRangeSearchCursor.java
@@ -69,7 +69,7 @@
         }
         lsmHarness = lsmInitState.getLSMHarness();
         operationalComponents = lsmInitState.getOperationalComponents();
-        includeMutableComponent = lsmInitState.getIncludeMemComponent();
+        setIncludeMutableComponent(lsmInitState.getIncludeMemComponent());
 
         // For searching the deleted-keys BTrees.
         this.keysOnlyTuple = lsmInitState.getKeysOnlyTuple();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
index 2520eac..678d4bc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesSearchCursor.java
@@ -80,18 +80,18 @@
         rtreeSearchPredicate = (SearchPredicate) searchPred;
         searchCallback = lsmInitialState.getSearchOperationCallback();
 
-        includeMutableComponent = false;
+        setIncludeMutableComponent(false);
         numMutableComponents = 0;
         int numImmutableComponents = 0;
         for (ILSMComponent component : operationalComponents) {
             if (component.getType() == LSMComponentType.MEMORY) {
-                includeMutableComponent = true;
+                setIncludeMutableComponent(true);
                 numMutableComponents++;
             } else {
                 numImmutableComponents++;
             }
         }
-        if (includeMutableComponent) {
+        if (isIncludeMutableComponent()) {
             btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
         }
 
@@ -140,7 +140,7 @@
 
     @Override
     public boolean hasNext() throws HyracksDataException {
-        if (includeMutableComponent) {
+        if (isIncludeMutableComponent()) {
             if (foundNext) {
                 return true;
             }
@@ -234,7 +234,7 @@
         }
         currentCursor = 0;
         foundNext = false;
-        if (includeMutableComponent) {
+        if (isIncludeMutableComponent()) {
             for (int i = 0; i < numMutableComponents; i++) {
                 mutableRTreeCursors[i].reset();
                 btreeCursors[i].reset();
@@ -248,7 +248,7 @@
         if (!open) {
             return;
         }
-        if (includeMutableComponent) {
+        if (isIncludeMutableComponent()) {
             for (int i = 0; i < numMutableComponents; i++) {
                 mutableRTreeCursors[i].close();
                 btreeCursors[i].close();

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2243
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic08a9372c608d6de960e1419899530aa55aa72e0
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <ba...@gmail.com>