You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2016/01/08 18:54:31 UTC
incubator-asterixdb-hyracks git commit: Fix for ASTERIXDB-1247
Repository: incubator-asterixdb-hyracks
Updated Branches:
refs/heads/master a17fa8a9d -> 45d1649c8
Fix for ASTERIXDB-1247
It seems like the root of this is the testing harness closing the index/deleting the file before it has
had a chance to flush all of its pages. There are also some changes to cover potential
corner cases where confiscated pages could be lost, but this doesn't seem to directly
affect the bug.
Change-Id: Ia580242b3f7753fc2f793f879332de3270ee3fee
Reviewed-on: https://asterix-gerrit.ics.uci.edu/575
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/45d1649c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/45d1649c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/45d1649c
Branch: refs/heads/master
Commit: 45d1649c8c510a6304e19fc9e5ed3481ea949618
Parents: a17fa8a
Author: Ian Maxon <im...@apache.org>
Authored: Fri Jan 8 01:03:31 2016 -0800
Committer: Ian Maxon <im...@apache.org>
Committed: Fri Jan 8 09:34:56 2016 -0800
----------------------------------------------------------------------
.../hyracks/storage/am/btree/impls/BTree.java | 27 ++++++++------------
.../am/common/impls/AbstractTreeIndex.java | 10 ++++++--
.../hyracks/storage/am/rtree/impls/RTree.java | 6 ++---
.../storage/common/buffercache/BufferCache.java | 3 +--
.../common/buffercache/FIFOLocalWriter.java | 25 ++++++++----------
.../common/buffercache/IFIFOPageWriter.java | 4 +--
.../am/btree/OrderedIndexExamplesTest.java | 3 ++-
7 files changed, 36 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 7099041..08f98a3 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -959,7 +959,6 @@ public class BTree extends AbstractTreeIndex {
public class BTreeBulkLoader extends AbstractTreeIndex.AbstractTreeIndexBulkLoader {
protected final ISplitKey splitKey;
protected final boolean verifyInput;
- protected List<ICachedPage> pagesToWrite;
public BTreeBulkLoader(float fillFactor, boolean verifyInput, boolean appendOnly) throws TreeIndexException,
HyracksDataException {
@@ -967,7 +966,6 @@ public class BTree extends AbstractTreeIndex {
this.verifyInput = verifyInput;
splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
- pagesToWrite = new ArrayList<ICachedPage>();
}
@Override
@@ -1003,15 +1001,16 @@ public class BTree extends AbstractTreeIndex {
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
splitKey.setLeftPage(leafFrontier.pageId);
- pagesToWrite.clear();
propagateBulk(1, pagesToWrite);
leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
((IBTreeLeafFrame) leafFrame).setNextLeaf(leafFrontier.pageId);
+
queue.put(leafFrontier.page);
for (ICachedPage c : pagesToWrite) {
queue.put(c);
}
+ pagesToWrite.clear();
splitKey.setRightPage(leafFrontier.pageId);
leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId,
@@ -1027,13 +1026,7 @@ public class BTree extends AbstractTreeIndex {
leafFrame.setPage(leafFrontier.page);
((IBTreeLeafFrame) leafFrame).insertSorted(tuple);
- } catch (IndexException e) {
- handleException();
- throw e;
- } catch (HyracksDataException e) {
- handleException();
- throw e;
- } catch (RuntimeException e) {
+ } catch (IndexException | HyracksDataException | RuntimeException e) {
handleException();
throw e;
}
@@ -1124,14 +1117,14 @@ public class BTree extends AbstractTreeIndex {
}
@Override
- protected void handleException() throws HyracksDataException {
- super.handleException();
- }
-
- @Override
public void end() throws HyracksDataException {
- persistFrontiers(0, -1);
- super.end();
+ try{
+ persistFrontiers(0, -1);
+ super.end();
+ } catch ( HyracksDataException | RuntimeException e) {
+ handleException();
+ throw e;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 015404d..58ca2d5 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.storage.am.common.impls;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -317,6 +318,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
protected boolean releasedLatches;
public boolean appendOnly = false;
protected final IFIFOPageQueue queue;
+ protected List<ICachedPage> pagesToWrite;
public AbstractTreeIndexBulkLoader(float fillFactor, boolean appendOnly) throws TreeIndexException,
HyracksDataException {
@@ -346,8 +348,8 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference());
leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
- leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle
- .getDiskPageId(fileId, leafFrontier.pageId));
+ leafFrontier.page = bufferCache.confiscatePage(
+ BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId));
interiorFrame.setPage(leafFrontier.page);
interiorFrame.initBuffer((byte) 0);
@@ -359,6 +361,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
slotSize = leafFrame.getSlotSize();
nodeFrontiers.add(leafFrontier);
+ pagesToWrite = new ArrayList<>();
}
public abstract void add(ITupleReference tuple) throws IndexException, HyracksDataException;
@@ -371,6 +374,9 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
bufferCache.returnPage(frontierPage,false);
}
}
+ for(ICachedPage pageToDiscard: pagesToWrite){
+ bufferCache.returnPage(pageToDiscard, false);
+ }
releasedLatches = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index ae57071..d01bad2 100644
--- a/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -883,7 +883,6 @@ public class RTree extends AbstractTreeIndex {
ITreeIndexTupleReference mbrTuple = interiorFrame.createTupleReference();
ByteBuffer mbr;
List<Integer> prevNodeFrontierPages = new ArrayList<Integer>();
- List<ICachedPage> pagesToWrite = new ArrayList<ICachedPage>();
public RTreeBulkLoader(float fillFactor, boolean appendOnly) throws TreeIndexException, HyracksDataException {
super(fillFactor, appendOnly);
@@ -918,7 +917,6 @@ public class RTree extends AbstractTreeIndex {
} else {
prevNodeFrontierPages.set(0, leafFrontier.pageId);
}
- pagesToWrite.clear();
propagateBulk(1, false, pagesToWrite);
leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
@@ -926,6 +924,8 @@ public class RTree extends AbstractTreeIndex {
for (ICachedPage c : pagesToWrite) {
queue.put(c);
}
+
+ pagesToWrite.clear();
leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId,
leafFrontier.pageId));
leafFrame.setPage(leafFrontier.page);
@@ -1082,4 +1082,4 @@ public class RTree extends AbstractTreeIndex {
public void validate() throws HyracksDataException {
throw new UnsupportedOperationException("Validation not implemented for R-Trees.");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index f66726b..75adfc9 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -546,7 +546,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
}
}
- private void write(CachedPage cPage) throws HyracksDataException {
+ void write(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
//synchronize on fInfo to prevent the file handle from being deleted until the page is written.
synchronized (fInfo) {
@@ -1196,5 +1196,4 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
}
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
index 2f66b15..b5d0ac7 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
@@ -30,25 +30,20 @@ public class FIFOLocalWriter implements IFIFOPageWriter {
}
@Override
- public void write(ICachedPage page, IBufferCache ibufferCache) throws HyracksDataException {
- BufferCache bufferCache = (BufferCache)ibufferCache;
+ public void write(ICachedPage page, BufferCache bufferCache) throws HyracksDataException {
CachedPage cPage = (CachedPage)page;
- BufferedFileHandle fInfo = bufferCache.getFileInfo(cPage);
- if (fInfo.fileHasBeenDeleted()) {
- return;
+ try {
+ bufferCache.write(cPage);
+ } finally {
+ bufferCache.returnPage(cPage);
+ if (DEBUG) {
+ System.out.println("[FIFO] Return page: " + cPage.cpid + "," + cPage.dpid);
+ }
}
- cPage.buffer.position(0);
- cPage.buffer.limit(bufferCache.getPageSize());
- bufferCache.ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * bufferCache.getPageSize(),
- cPage.buffer);
- bufferCache.returnPage(cPage);
- if(DEBUG) System.out.println("[FIFO] Return page: " + cPage.cpid + "," + cPage.dpid);
}
@Override
- public void sync(int fileId, IBufferCache ibufferCache) throws HyracksDataException {
- BufferCache bufferCache = (BufferCache)ibufferCache;
- BufferedFileHandle fInfo = bufferCache.getFileInfo(fileId);
- bufferCache.ioManager.sync(fInfo.getFileHandle(), true);
+ public void sync(int fileId, BufferCache bufferCache) throws HyracksDataException {
+ bufferCache.force(fileId,true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
index 2281a44..7380261 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
@@ -19,7 +19,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IFIFOPageWriter {
- public void write(ICachedPage page, IBufferCache bufferCache) throws HyracksDataException;
+ public void write(ICachedPage page, BufferCache bufferCache) throws HyracksDataException;
- void sync(int fileId, IBufferCache ibufferCache) throws HyracksDataException;
+ void sync(int fileId, BufferCache bufferCache) throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/45d1649c/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index 8c01105..94606ce 100644
--- a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -25,6 +25,7 @@ import java.util.Random;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.junit.Test;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -723,6 +724,7 @@ public abstract class OrderedIndexExamplesTest {
fail("Unexpected exception: " + e.getMessage());
}
// Success.
+
break;
} catch (TreeIndexDuplicateKeyException e2) {
if (j != i) {
@@ -732,7 +734,6 @@ public abstract class OrderedIndexExamplesTest {
break;
}
}
-
treeIndex.deactivate();
treeIndex.destroy();
}