You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2016/06/11 02:27:23 UTC
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/920
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj
Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart
- large page information managed by buffer cache, outside of page
- size header written out a prefix to pages
Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
54 files changed, 530 insertions(+), 419 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/920/1
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
index 7d92ce2..9fb4e94 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.common.config;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
public class AsterixStorageProperties extends AbstractAsterixProperties {
private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize";
@@ -62,7 +64,7 @@
}
public int getBufferCacheNumPages() {
- return (int) (getBufferCacheSize() / getBufferCachePageSize());
+ return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
}
public int getBufferCacheMaxOpenFiles() {
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
similarity index 66%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
index 22bec97..77c8179 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
@@ -16,10 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+/*
+ * Test case Name : big_object_bulkload.aql
+ * Description : bulkload insert of large objects
+ * Expected Result : Success
+ * Date : 20th April 2016
+ */
-public interface ILargePageHelper {
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use dataverse testdv2;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
+create type testtype as closed {
+ id: int64,
+ name: string,
+ hobbies: {{string}}
}
+
+create dataset testds(testtype) primary key id;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
similarity index 74%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
index 22bec97..f1b3819 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
@@ -16,10 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+/**
+ *
+ * Big object (20 MB) loading test
+ * Expected result: success
+ *
+ */
-public interface ILargePageHelper {
+use dataverse testdv2;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
-}
+load dataset testds
+using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
similarity index 79%
rename from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
rename to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
index 22bec97..23c1bed 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
@@ -16,10 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+use dataverse testdv2;
-public interface ILargePageHelper {
-
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
-}
+for $d in dataset("testds")
+where $d.id = 1
+return $d
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
new file mode 100644
index 0000000..5acf078
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
@@ -0,0 +1 @@
+{ "id": 1, "name": "Person One", "hobbies": {{ "%lorembytes:20971520%" }} }
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
new file mode 100755
index 0000000..37bc7b0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
@@ -0,0 +1,2 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9;
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
new file mode 100755
index 0000000..5f247db
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
new file mode 100755
index 0000000..e2b0583
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
@@ -0,0 +1,2 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 08b8e87..75b2457 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -28,6 +28,11 @@
<output-dir compare="Text">external_index</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="query_after_restart">
+ <compilation-unit name="big_object_20M">
+ <output-dir compare="Text">big_object_20M</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="recover_after_abort">
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 47eac7e..e355968 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -43,8 +43,12 @@
public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
+ public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
+
public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
+ public long syncRead(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
+
public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data);
public IIOFuture asyncRead(IFileHandle fHandle, long offset, ByteBuffer data);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 519a9d7..c5196fe 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -22,6 +22,7 @@
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -49,7 +50,7 @@
public IOManager(List<IODeviceHandle> devices) throws HyracksException {
this.ioDevices = Collections.unmodifiableList(devices);
- workAreaIODevices = new ArrayList<IODeviceHandle>();
+ workAreaIODevices = new ArrayList<>();
for (IODeviceHandle d : ioDevices) {
if (d.getWorkAreaPath() != null) {
new File(d.getPath(), d.getWorkAreaPath()).mkdirs();
@@ -109,13 +110,48 @@
}
}
+ @Override
+ public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException {
+ try {
+ if (fHandle == null) {
+ throw new IllegalStateException("Trying to write to a deleted file.");
+ }
+ int n = 0;
+ int remaining = 0;
+ for (ByteBuffer buf : dataArray) {
+ remaining += buf.remaining();
+ }
+ final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel();
+ while (remaining > 0) {
+ long len;
+ synchronized (fileChannel) {
+ fileChannel.position(offset);
+ len = fileChannel.write(dataArray);
+ }
+ if (len < 0) {
+ throw new HyracksDataException(
+ "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString());
+ }
+ remaining -= len;
+ offset += len;
+ n += len;
+ }
+ return n;
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
/**
* Please do check the return value of this read!
*
* @param fHandle
* @param offset
* @param data
- * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's current size
+ * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's
+ * current size
* @throws HyracksDataException
*/
@Override
@@ -140,6 +176,46 @@
}
}
+ /**
+ * Please do check the return value of this read!
+ *
+ * @param fHandle
+ * @param offset
+ * @param buffers
+ * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's
+ * current size
+ * @throws HyracksDataException
+ */
+ @Override
+ public long syncRead(IFileHandle fHandle, long offset, ByteBuffer[] buffers) throws HyracksDataException {
+ try {
+ int n = 0;
+ int remaining = 0;
+ for (ByteBuffer buf : buffers) {
+ remaining += buf.remaining();
+ }
+ final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel();
+ while (remaining > 0) {
+ long len;
+ synchronized (fileChannel) {
+ fileChannel.position(offset);
+ len = fileChannel.read(buffers);
+ }
+ if (len < 0) {
+ return n == 0 ? -1 : n;
+ }
+ remaining -= len;
+ offset += len;
+ n += len;
+ }
+ return n;
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
@Override
public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data) {
AsyncWriteRequest req = new AsyncWriteRequest((FileHandle) fHandle, offset, data);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
index 1081dc8..2884748 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
@@ -38,5 +38,7 @@
public void setLargeFlag(boolean largePage);
+ public boolean isLargePage();
+
public void validate(PageValidationInfo pvi) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index 7648ee6..2e072ce 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -21,14 +21,13 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public interface IBTreeLeafFrame extends IBTreeFrame {
public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp,
@@ -53,12 +52,6 @@
public int getNextLeaf();
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId);
-
- public boolean isLargePage();
-
- public int getSupplementalNumPages();
-
- void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
- IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException;
+ void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index 701f44f..7c34cce 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -33,11 +33,9 @@
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixPrefixTupleReference;
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixTupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameCompressor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -51,7 +49,7 @@
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
/**
* WARNING: only works when tupleWriter is an instance of TypeAwareTupleWriter
@@ -537,11 +535,6 @@
}
@Override
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId) {
- throw new IllegalStateException();
- }
-
- @Override
public void setLargeFlag(boolean largePage) {
throw new IllegalStateException();
}
@@ -549,11 +542,6 @@
@Override
public boolean isLargePage() {
return false;
- }
-
- @Override
- public int getSupplementalNumPages() {
- throw new IllegalStateException();
}
public int getPrefixTupleCount() {
@@ -596,7 +584,8 @@
}
@Override
- public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException {
BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame;
@@ -804,12 +793,7 @@
}
@Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
-
- @Override
- public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException {
+ public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper extraPageBlockHelper) throws HyracksDataException {
throw new IllegalStateException("nyi");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
index c558d30..f7b270c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
deleted file mode 100644
index 79f7f53..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.btree.frames;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.storage.common.buffercache.CachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
-
-public class BTreeLargeFrameHelper implements ILargePageHelper {
-
- public static final BTreeLargeFrameHelper INSTANCE = new BTreeLargeFrameHelper();
-
- private BTreeLargeFrameHelper() {
- }
-
- @Override
- public int getSupplementalBlockNumPages(CachedPage cPage) {
- ByteBuffer buffer = cPage.getBuffer();
- return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalNumPages(buffer) : 0;
- }
-
- @Override
- public int getSupplementalBlockPageId(CachedPage cPage) {
- ByteBuffer buffer = cPage.getBuffer();
- return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalPageId(buffer) : -1;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index c2b18b7..9505f8a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -29,10 +29,8 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -43,6 +41,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame {
@@ -55,7 +54,7 @@
private MultiComparator cmp;
public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
- super(tupleWriter, new OrderedSlotManager(), null);
+ super(tupleWriter, new OrderedSlotManager());
cmpFrameTuple = tupleWriter.createTupleReference();
previousFt = tupleWriter.createTupleReference();
}
@@ -201,7 +200,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException, TreeIndexException {
ByteBuffer right = rightFrame.getBuffer();
int tupleCount = getTupleCount();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
index 3737486..4ecb83d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 8f560fe..dbea5fd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -25,10 +25,8 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -40,25 +38,23 @@
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
protected static final int nextLeafOff = flagOff + 1; // 22
- protected static final int supplementalNumPagesOff = nextLeafOff + 4; // 26
- protected static final int supplementalPageIdOff = supplementalNumPagesOff + 4; // 30
private MultiComparator cmp;
private final ITreeIndexTupleReference previousFt;
- public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, ILargePageHelper largePageHelper) {
- super(tupleWriter, new OrderedSlotManager(), largePageHelper);
+ public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
previousFt = tupleWriter.createTupleReference();
}
@Override
public int getPageHeaderSize() {
- return supplementalPageIdOff + 4;
+ return nextLeafOff + 4;
}
@Override
@@ -70,8 +66,6 @@
public void initBuffer(byte level) {
super.initBuffer(level);
buf.putInt(nextLeafOff, -1);
- buf.putInt(supplementalNumPagesOff, 0);
- buf.putInt(supplementalPageIdOff, -1);
}
@Override
@@ -82,28 +76,6 @@
@Override
public int getNextLeaf() {
return buf.getInt(nextLeafOff);
- }
-
- public static int getSupplementalNumPages(ByteBuffer buf) {
- return buf.getInt(supplementalNumPagesOff);
- }
-
- public int getSupplementalNumPages() {
- return getSupplementalNumPages(buf);
- }
-
- public static int getSupplementalPageId(ByteBuffer buf) {
- return buf.getInt(supplementalPageIdOff);
- }
-
- public int getSupplementalPageId() {
- return getSupplementalPageId(buf);
- }
-
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId) {
- setLargeFlag(true);
- buf.putInt(supplementalNumPagesOff, supplementalPages);
- buf.putInt(supplementalPageIdOff, supplementalBlockPageId);
}
@Override
@@ -204,7 +176,7 @@
boolean isLargeTuple(int tupleSize) {
// TODO(mblow): make page size available to avoid calculating it
- int pageSize = isLargePage() ? buf.capacity() / (getSupplementalNumPages() + 1) : buf.capacity();
+ int pageSize = isLargePage() ? buf.capacity() / page.getFrameSizeMultiplier() : buf.capacity();
return tupleSize > getMaxTupleSize(pageSize);
}
@@ -237,8 +209,8 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
- throws HyracksDataException {
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
+ throws HyracksDataException {
int tupleSize = getBytesRequiredToWriteTuple(tuple);
@@ -249,7 +221,7 @@
// Find split point, and determine into which frame the new tuple should
// be inserted into.
- BTreeNSMLeafFrame targetFrame = null;
+ BTreeNSMLeafFrame targetFrame;
frameTuple.resetByTupleIndex(this, tupleCount - 1);
if (cmp.compare(tuple, frameTuple) > 0) {
// This is a special optimization case when the tuple to be inserted is the largest key on the page.
@@ -275,22 +247,14 @@
targetFrame = this;
}
int tuplesToRight = tupleCount - tuplesToLeft;
- int supplementalPages = 0;
- int supplementalPageId = -1;
if (isLargePage()) {
- ((BTreeNSMLeafFrame) rightFrame).growCapacity(freePageManager, metaFrame, bufferCache,
+ ((BTreeNSMLeafFrame) rightFrame).growCapacity(extraPageBlockHelper, bufferCache,
buf.capacity() - rightFrame.getBuffer().capacity());
- supplementalPages = ((BTreeNSMLeafFrame) rightFrame).getSupplementalNumPages();
- supplementalPageId = ((BTreeNSMLeafFrame) rightFrame).getSupplementalPageId();
}
ByteBuffer right = rightFrame.getBuffer();
// Copy entire page.
System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity());
- if (isLargePage()) {
- // restore the supplemental page metadata
- ((BTreeNSMLeafFrame) rightFrame).configureLargePage(supplementalPages, supplementalPageId);
- }
// On the right page we need to copy rightmost slots to the left.
int src = rightFrame.getSlotManager().getSlotEndOff();
@@ -309,7 +273,7 @@
}
if (tupleLarge) {
- targetFrame.ensureCapacity(freePageManager, metaFrame, bufferCache, tuple);
+ targetFrame.ensureCapacity(bufferCache, tuple, extraPageBlockHelper);
}
// Insert the new tuple.
@@ -317,7 +281,7 @@
// it's safe to catch this exception since it will have been caught
// before reaching here
try {
- targetTupleIndex = ((BTreeNSMLeafFrame) targetFrame).findInsertTupleIndex(tuple);
+ targetTupleIndex = targetFrame.findInsertTupleIndex(tuple);
} catch (TreeIndexException e) {
throw new IllegalStateException(e);
}
@@ -332,13 +296,13 @@
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
- public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
- IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException {
+ public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple,
+ IExtraPageBlockHelper extraPageBlockHelper) throws HyracksDataException {
int gapBytes = getBytesRequiredToWriteTuple(tuple) - getFreeContiguousSpace();
- growCapacity(freePageManager, metaFrame, bufferCache, gapBytes);
+ growCapacity(extraPageBlockHelper, bufferCache, gapBytes);
}
- public void growCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
+ public void growCapacity(IExtraPageBlockHelper extraPageBlockHelper,
IBufferCache bufferCache, int delta) throws HyracksDataException {
if (delta <= 0) {
setLargeFlag(true);
@@ -346,35 +310,21 @@
}
int deltaPages = (int) Math.ceil((double) delta / bufferCache.getPageSize());
int framePagesOld = getBuffer().capacity() / bufferCache.getPageSize();
- int oldSupplementalPages = 0;
- int oldSupplementalPageId = -1;
- if (isLargePage()) {
- oldSupplementalPages = getSupplementalNumPages();
- oldSupplementalPageId = getSupplementalPageId();
- }
-
- configureLargePage(framePagesOld + deltaPages - 1,
- freePageManager.getFreePageBlock(metaFrame, framePagesOld + deltaPages - 1));
-
- int pageDelta = (framePagesOld + deltaPages) - 1 - oldSupplementalPages;
+ int newMultiplier = framePagesOld + deltaPages;
// we need to get the old slot offsets before we grow
int oldSlotEnd = slotManager.getSlotEndOff();
int oldSlotStart = slotManager.getSlotStartOff() + slotManager.getSlotSize();
- bufferCache.resizePage(getPage(), framePagesOld + deltaPages);
- buf = getPage().getBuffer();
+ bufferCache.resizePage(getPage(), newMultiplier, extraPageBlockHelper);
- // return the dropped supplemental pages to the page manager...
- if (oldSupplementalPages > 0) {
- freePageManager.addFreePageBlock(metaFrame, oldSupplementalPageId, oldSupplementalPages);
- }
+ buf = getPage().getBuffer();
// fixup the slots
System.arraycopy(buf.array(), oldSlotEnd, buf.array(), slotManager.getSlotEndOff(), oldSlotStart - oldSlotEnd);
// fixup total free space counter
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * pageDelta));
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * deltaPages));
}
@Override
@@ -416,8 +366,6 @@
public String printHeader() {
StringBuilder strBuilder = new StringBuilder(super.printHeader());
strBuilder.append("nextLeafOff: " + nextLeafOff + "\n");
- strBuilder.append("supplementalNumPagesOff: " + supplementalNumPagesOff + "\n");
- strBuilder.append("supplementalPageIdOff: " + supplementalPageIdOff + "\n");
return strBuilder.toString();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
index 2b7f12b..3cf0ba5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -36,7 +35,7 @@
@Override
public IBTreeLeafFrame createFrame() {
- return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), getLargePageHelper());
+ return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
@Override
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return BTreeLargeFrameHelper.INSTANCE;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 3043940..5995004 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -66,7 +66,6 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.apache.hyracks.storage.common.buffercache.CachedPage;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
@@ -103,8 +102,7 @@
RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp);
int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
int currentPageId = bulkloadLeafStart;
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false,
- largePageHelper);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
try {
cursor.setBufferCache(bufferCache);
@@ -136,7 +134,7 @@
}
private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(page);
PageValidationInfo currentPvi = ctx.validationInfos.peekFirst();
@@ -220,8 +218,7 @@
ICachedPage originalPage = ctx.interiorFrame.getPage();
for (int i = 0; i < ctx.smPages.size(); i++) {
int pageId = ctx.smPages.get(i);
- ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false,
- largePageHelper);
+ ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
smPage.acquireWriteLatch();
try {
ctx.interiorFrame.setPage(smPage);
@@ -246,17 +243,17 @@
private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// Make sure the root is always in the same page.
ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()),
- false, largePageHelper);
+ false);
leftNode.acquireWriteLatch();
try {
int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true,
- largePageHelper);
+ ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
newLeftNode.acquireWriteLatch();
try {
boolean largePage = false;
if (leftNode.getBuffer().capacity() > newLeftNode.getBuffer().capacity()) {
- bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize());
+ bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize(),
+ ctx);
largePage = true;
}
// Copy left child to new left child.
@@ -268,7 +265,7 @@
long leftNodeLSN = ctx.interiorFrame.getPageLsn();
// Initialize new root (leftNode becomes new root).
if (largePage) {
- bufferCache.resizePage(leftNode, 1);
+ bufferCache.resizePage(leftNode, 1, ctx);
ctx.interiorFrame.setPage(leftNode);
ctx.interiorFrame.setLargeFlag(false);
} else {
@@ -372,7 +369,7 @@
switch (spaceStatus) {
case EXPAND: {
// TODO: avoid repeated calculation of tuple size
- ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple);
+ ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx);
}
// fall-through
case SUFFICIENT_CONTIGUOUS_SPACE: {
@@ -432,8 +429,7 @@
}
}
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true,
- largePageHelper);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
@@ -449,7 +445,7 @@
} else {
ctx.modificationCallback.found(null, tuple);
}
- ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, bufferCache);
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache);
ctx.smPages.add(pageId);
ctx.smPages.add(rightPageId);
@@ -492,7 +488,7 @@
ctx.modificationCallback.found(beforeTuple, tuple);
ctx.leafFrame.delete(tuple, oldTupleIndex);
ctx.leafFrame.compact();
- ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple);
+ ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx);
int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
ctx.leafFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
@@ -550,8 +546,7 @@
switch (spaceStatus) {
case INSUFFICIENT_SPACE: {
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true,
- largePageHelper);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeFrame rightFrame = ctx.createInteriorFrame();
@@ -560,8 +555,7 @@
rightFrame.setMultiComparator(ctx.cmp);
// instead of creating a new split key, use the existing
// splitKey
- ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, freePageManager,
- ctx.metaFrame, bufferCache);
+ ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, ctx, bufferCache);
ctx.smPages.add(pageId);
ctx.smPages.add(rightPageId);
ctx.interiorFrame.setSmFlag(true);
@@ -631,7 +625,7 @@
}
private ICachedPage isConsistent(int pageId, BTreeOpContext ctx) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
ctx.interiorFrame.setPage(node);
boolean isConsistent = ctx.pageLsns.getLast() == ctx.interiorFrame.getPageLsn();
@@ -645,7 +639,7 @@
private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx)
throws HyracksDataException, TreeIndexException {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(node);
// this check performs an unprotected read in the page
// the following could happen: TODO fill out
@@ -707,7 +701,7 @@
// Is there a propagated split key?
if (ctx.splitKey.getBuffer() != null) {
ICachedPage interiorNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ BufferedFileHandle.getDiskPageId(fileId, pageId), false);
interiorNode.acquireWriteLatch();
try {
// Insert or update op. Both can cause split keys to propagate upwards.
@@ -839,8 +833,7 @@
private BTreeOpContext createOpContext(IIndexAccessor accessor, IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new BTreeOpContext(accessor, leafFrameFactory, interiorFrameFactory,
- freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback,
- searchCallback);
+ freePageManager, cmpFactories, modificationCallback, searchCallback);
}
@SuppressWarnings("rawtypes")
@@ -857,7 +850,7 @@
public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes,
StringBuilder strBuilder, MultiComparator cmp) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
try {
if (parent != null && unpin == true) {
@@ -1077,21 +1070,18 @@
int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize());
final int multiplier = (int) Math
.ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize));
- leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier);
- ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper);
+ if (multiplier > 1) {
+ leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier,
+ freePageManager.getFreePageBlock(metaFrame, multiplier - 1));
+ } else {
+ leafFrontier.page = bufferCache.confiscatePage(dpid);
+ }
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
- if (multiplier > 1) {
- int supplementalPages = multiplier - 1;
- ((IBTreeLeafFrame) leafFrame).configureLargePage(supplementalPages,
- freePageManager.getFreePageBlock(metaFrame, supplementalPages));
- } else {
- ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
- }
+ ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
} else {
final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
leafFrontier.page = bufferCache.confiscatePage(dpid);
- ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper);
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
}
@@ -1161,7 +1151,6 @@
propagateBulk(level + 1, pagesToWrite);
frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID);
- ((CachedPage) frontier.page).setLargePageHelper(largePageHelper);
interiorFrame.setPage(frontier.page);
interiorFrame.initBuffer((byte) level);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
index 974860e..2297cc5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.apache.hyracks.storage.am.common.api.ICursorInitialState;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
@@ -119,8 +118,7 @@
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
do {
- ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
if (exclusiveLatchNodes) {
nextLeaf.acquireWriteLatch();
page.releaseWriteLatch(isPageDirty);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
index 8513368..3649433 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
@@ -33,6 +33,7 @@
import org.apache.hyracks.storage.am.btree.api.ITupleAcceptor;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
@@ -43,8 +44,9 @@
import org.apache.hyracks.storage.am.common.ophelpers.IntArrayList;
import org.apache.hyracks.storage.am.common.ophelpers.LongArrayList;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
-public class BTreeOpContext implements IIndexOperationContext {
+public class BTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper {
private final int INIT_ARRAYLIST_SIZE = 6;
public IIndexAccessor accessor;
@@ -53,7 +55,8 @@
public ITreeIndexFrameFactory interiorFrameFactory;
public IBTreeLeafFrame leafFrame;
public IBTreeInteriorFrame interiorFrame;
- public ITreeIndexMetaDataFrame metaFrame;
+ public final IMetaDataPageManager freePageManager;
+ public final ITreeIndexMetaDataFrame metaFrame;
public IndexOperation op;
public ITreeIndexCursor cursor;
public BTreeCursorInitialState cursorInitialState;
@@ -75,9 +78,9 @@
public final ITreeIndexTupleReference leafFrameTuple;
public BTreeOpContext(IIndexAccessor accessor, ITreeIndexFrameFactory leafFrameFactory,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexMetaDataFrame metaFrame,
- IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ ITreeIndexFrameFactory interiorFrameFactory, IMetaDataPageManager freePageManager,
+ IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
this.accessor = accessor;
if (cmpFactories[0] != null) {
@@ -97,7 +100,8 @@
if (interiorFrame != null && this.cmp != null) {
interiorFrame.setMultiComparator(cmp);
}
- this.metaFrame = metaFrame;
+ this.freePageManager = freePageManager;
+ this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.pageLsns = new LongArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
this.smoCount = 0;
this.modificationCallback = modificationCallback;
@@ -233,4 +237,14 @@
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
}
+
+ @Override
+ public int getFreeBlock(int size) throws HyracksDataException {
+ return freePageManager.getFreePageBlock(metaFrame, size);
+ }
+
+ @Override
+ public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException {
+ freePageManager.addFreePageBlock(metaFrame, blockPageId, size);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 975bd9b..3301e37 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.util.TupleUtils;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.apache.hyracks.storage.am.common.api.ICursorInitialState;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
@@ -118,8 +117,7 @@
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
do {
- ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
if (exclusiveLatchNodes) {
nextLeaf.acquireWriteLatch();
page.releaseWriteLatch(isPageDirty);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
index 5b6cee9..7c2abb1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -38,7 +38,6 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeUtils {
@@ -49,7 +48,6 @@
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper();
IMetaDataPageManager freePageManager;
freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
@@ -63,7 +61,6 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper();
BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
cmpFactories, typeTraits.length, file);
return btree;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
index 711db9e..037c183 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -27,7 +27,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public interface ITreeIndexFrame {
@@ -73,7 +73,7 @@
public String printHeader();
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException, TreeIndexException;
public ISlotManager getSlotManager();
@@ -104,6 +104,4 @@
public ITreeIndexTupleReference createTupleReference();
public void setMultiComparator(MultiComparator cmp);
-
- ILargePageHelper getLargePageHelper();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
index 248baf5..8a705dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
@@ -20,10 +20,7 @@
import java.io.Serializable;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
-
public interface ITreeIndexFrameFactory extends Serializable {
ITreeIndexFrame createFrame();
ITreeIndexTupleWriterFactory getTupleWriterFactory();
- ILargePageHelper getLargePageHelper();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
index ea6bcac..fe6ddca 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
@@ -31,7 +31,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
@@ -51,14 +50,12 @@
protected ITreeIndexTupleWriter tupleWriter;
protected ITreeIndexTupleReference frameTuple;
- protected ILargePageHelper largePageHelper;
- public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager, ILargePageHelper largePageHelper) {
+ public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) {
this.tupleWriter = tupleWriter;
this.frameTuple = tupleWriter.createTupleReference();
this.slotManager = slotManager;
this.slotManager.setFrame(this);
- this.largePageHelper = largePageHelper;
}
@Override
@@ -342,8 +339,4 @@
return buf.capacity() - getFreeSpaceOff() - (getTupleCount() * slotManager.getSlotSize());
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return largePageHelper;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index b7b39d4..3a7eb77 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -26,12 +26,20 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.IndexException;
+import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -62,8 +70,6 @@
protected int bulkloadLeafStart = 0;
- protected final ILargePageHelper largePageHelper;
-
public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetaDataPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
@@ -78,7 +84,6 @@
this.cmpFactories = cmpFactories;
this.fieldCount = fieldCount;
this.file = file;
- this.largePageHelper = leafFrameFactory.getLargePageHelper();
}
public synchronized void create() throws HyracksDataException {
@@ -125,8 +130,7 @@
ITreeIndexFrame frame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
freePageManager.init(metaFrame, rootPage);
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
rootNode.acquireWriteLatch();
try {
frame.setPage(rootNode);
@@ -244,8 +248,7 @@
if(freePageManager.appendOnlyMode() && bufferCache.getNumPagesOfFile(fileId) <= MINIMAL_TREE_PAGE_COUNT){
return true;
}
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
try {
frame.setPage(rootNode);
@@ -263,8 +266,7 @@
public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException {
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
try {
frame.setPage(rootNode);
@@ -394,13 +396,12 @@
//move the root page to the first data page if necessary
bufferCache.finishQueue();
if (!appendOnly) {
- ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true,
- largePageHelper);
+ ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
newRoot.acquireWriteLatch();
//root will be the highest frontier
NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1);
ICachedPage oldRoot = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false, largePageHelper);
+ BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false);
oldRoot.acquireReadLatch();
lastNodeFrontier.page = oldRoot;
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
index 87449eb..679ea2f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
@@ -64,14 +64,15 @@
}
private boolean positionToNextLeaf(boolean skipCurrent) throws HyracksDataException {
- while ((frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) && (currentPageId <= maxPageId)) {
- currentPageId++;
+ while (frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) {
+ if (++currentPageId > maxPageId) {
+ break;
+ }
page.releaseReadLatch();
bufferCache.unpin(page);
- ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false,
- frame.getLargePageHelper());
+ ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
nextPage.acquireReadLatch();
page = nextPage;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index d9013d3..5668992 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -53,8 +53,7 @@
// scan entire file to determine pages in each level
int maxPageId = freePageManager.getMaxPage(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
try {
frame.setPage(page);
@@ -89,8 +88,7 @@
int pageId = remainingPageIds[index];
// pin & latch then immediately unlatch & unpin
- ICachedPage page = bufferCache.pin(BufferedFileHandle
- .getDiskPageId(fileId, pageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
page.releaseReadLatch();
bufferCache.unpin(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index 7a9e588..712acb8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -52,8 +52,7 @@
int maxPageId = freePageManager.getMaxPage(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false, leafFrame.getLargePageHelper());
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
try {
metaFrame.setPage(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index c4fce24..748dbfe 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -25,8 +25,8 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.replication.IIOReplicationManager;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -69,11 +69,6 @@
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
return vbc.pin(dpid, newPage);
- }
-
- @Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- return vbc.pin(dpid, newPage, helper);
}
@Override
@@ -163,8 +158,9 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return vbc.confiscateLargePage(dpid, multiplier);
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ return vbc.confiscateLargePage(dpid, multiplier, extraBlockPageId);
}
@Override
@@ -218,7 +214,8 @@
}
@Override
- public void resizePage(ICachedPage page, int multiple) {
- vbc.resizePage(page, multiple);
+ public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ vbc.resizePage(page, multiplier, extraPageBlockHelper);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index e62f824..40c7734 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -34,8 +34,8 @@
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.buffercache.IQueueInfo;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -66,7 +66,7 @@
this.numPages = 2 * (numPages / 2) + 1;
buckets = new CacheBucket[this.numPages];
- pages = new ArrayList<VirtualPage>();
+ pages = new ArrayList<>();
nextFree = 0;
largePages = new AtomicInteger(0);
open = false;
@@ -168,11 +168,6 @@
}
@Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- return pin(dpid, newPage);
- }
-
- @Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
VirtualPage page = null;
int hash = hash(dpid);
@@ -213,6 +208,7 @@
synchronized (pages) {
if (nextFree >= pages.size()) {
page = new VirtualPage(allocator.allocate(pageSize, 1)[0]);
+ page.multiplier = 1;
pages.add(page);
} else {
page = pages.get(nextFree);
@@ -224,9 +220,9 @@
}
@Override
- public void resizePage(ICachedPage cPage, int multiplier) {
+ public void resizePage(ICachedPage cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) {
ByteBuffer oldBuffer = cPage.getBuffer();
- int origMultiplier = oldBuffer.capacity() / pageSize;
+ int origMultiplier = cPage.getFrameSizeMultiplier();
if (origMultiplier == multiplier) {
// no-op
return;
@@ -253,6 +249,7 @@
largePages.getAndAdd(multiplier - origMultiplier);
}
((VirtualPage)cPage).buffer = newBuffer;
+ ((VirtualPage)cPage).multiplier = multiplier;
}
@Override
@@ -354,6 +351,7 @@
ByteBuffer buffer;
final ReadWriteLock latch;
volatile long dpid;
+ int multiplier;
VirtualPage next;
public VirtualPage(ByteBuffer buffer) {
@@ -406,6 +404,10 @@
throw new UnsupportedOperationException();
}
+ @Override
+ public int getFrameSizeMultiplier() {
+ return multiplier;
+ }
}
//These 4 methods aren't applicable here.
@@ -447,12 +449,13 @@
}
@Override
- public ICachedPage confiscatePage(long dpid) {
+ public ICachedPage confiscatePage(long dpid) throws HyracksDataException {
throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index 56827a5..84cb82e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -21,11 +21,9 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
@@ -34,6 +32,7 @@
import org.apache.hyracks.storage.am.rtree.api.IRTreePolicy;
import org.apache.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeFrame {
protected static final int pageNsnOff = flagOff + 1; // 22
@@ -50,7 +49,7 @@
public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders,
RTreePolicyType rtreePolicyType, boolean isPointMBR) {
- super(tupleWriter, new UnorderedSlotManager(), null);
+ super(tupleWriter, new UnorderedSlotManager());
this.mbrTuples = new ITreeIndexTupleReference[keyValueProviders.length];
for (int i = 0; i < keyValueProviders.length; i++) {
this.mbrTuples[i] = tupleWriter.createTupleReference();
@@ -116,7 +115,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException {
rtreePolicy.split(this, buf, rightFrame, slotManager, frameTuple, tuple, splitKey);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
index 02177f4..8798a70 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
@@ -61,8 +60,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
index 6373c7e..8573119 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -61,8 +60,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index 84180aa..99edbc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -155,8 +155,8 @@
private RTreeOpContext createOpContext(IModificationOperationCallback modificationCallback) {
return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
- (IRTreeInteriorFrame) interiorFrameFactory.createFrame(),
- freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback);
+ (IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager, cmpFactories,
+ modificationCallback);
}
private void insert(ITupleReference tuple, IIndexOperationContext ictx)
@@ -412,7 +412,7 @@
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
- ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+ ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, ctx,
bufferCache);
ctx.interiorFrame.setRightPage(rightPageId);
} else {
@@ -421,7 +421,7 @@
rightFrame.initBuffer((byte) 0);
rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
ctx.modificationCallback.found(null, tuple);
- ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx,
bufferCache);
ctx.leafFrame.setRightPage(rightPageId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index 107dc30..a04562f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -22,8 +22,10 @@
import java.util.ArrayList;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
@@ -32,8 +34,9 @@
import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
-public class RTreeOpContext implements IIndexOperationContext {
+public class RTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper {
private static final int INITIAL_TRAVERSE_LIST_SIZE = 100;
private static final int INITIAL_HEIGHT = 8;
public final MultiComparator cmp;
@@ -42,7 +45,8 @@
public IndexOperation op;
public ITreeIndexCursor cursor;
public RTreeCursorInitialState cursorInitialState;
- public ITreeIndexMetaDataFrame metaFrame;
+ public final IMetaDataPageManager freePageManager;
+ public final ITreeIndexMetaDataFrame metaFrame;
public RTreeSplitKey splitKey;
public ITupleReference tuple;
// Used to record the pageIds and pageLsns of the visited pages.
@@ -56,8 +60,8 @@
public IModificationOperationCallback modificationCallback;
public RTreeOpContext(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories,
- IModificationOperationCallback modificationCallback) {
+ IMetaDataPageManager freePageManager, IBinaryComparatorFactory[] cmpFactories,
+ IModificationOperationCallback modificationCallback) {
if (cmpFactories[0] != null) {
this.cmp = MultiComparator.create(cmpFactories);
@@ -67,7 +71,8 @@
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
- this.metaFrame = metaFrame;
+ this.freePageManager = freePageManager;
+ this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.modificationCallback = modificationCallback;
pathList = new PathList(INITIAL_HEIGHT, INITIAL_HEIGHT);
NSNUpdates = new ArrayList<ICachedPage>();
@@ -121,4 +126,14 @@
public void setModificationCallback(IModificationOperationCallback modificationCallback) {
this.modificationCallback = modificationCallback;
}
+
+ @Override
+ public int getFreeBlock(int size) throws HyracksDataException {
+ return freePageManager.getFreePageBlock(metaFrame, size);
+ }
+
+ @Override
+ public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException {
+ freePageManager.addFreePageBlock(metaFrame, blockPageId, size);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 201526b..ceb8ca2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -28,8 +28,10 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@@ -49,6 +51,7 @@
import org.apache.hyracks.storage.common.file.IFileMapManager;
public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
+
private static final Logger LOGGER = Logger.getLogger(BufferCache.class.getName());
private static final int MAP_FACTOR = 3;
@@ -68,6 +71,8 @@
private final Map<Integer, BufferedFileHandle> fileInfoMap;
private final Set<Integer> virtualFiles;
private final AsyncFIFOPageQueueManager fifoWriter;
+ private final Queue<BufferCacheHeaderHelper> headerPageCache = new ConcurrentLinkedQueue<>();
+
//DEBUG
private ArrayList<CachedPage> confiscatedPages;
private Lock confiscateLock;
@@ -124,6 +129,10 @@
return pageSize;
}
+ public int getPageSizeWithHeader() {
+ return pageSize + RESERVED_HEADER_BYTES;
+ }
+
@Override
public int getNumPages() {
return pageReplacementStrategy.getMaxAllowedNumPages();
@@ -176,11 +185,6 @@
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
- return pin(dpid, newPage, null);
- }
-
- @Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
// Calling the pinSanityCheck should be used only for debugging, since
// the synchronized block over the fileInfoMap is a hot spot.
if (DEBUG) {
@@ -193,9 +197,7 @@
try {
for (CachedPage c : confiscatedPages) {
if (c.dpid == dpid && c.confiscated.get()) {
- while(confiscatedPages.contains(c)){
- throw new IllegalStateException();
- }
+ throw new IllegalStateException();
}
}
} finally{
@@ -206,7 +208,7 @@
// disk.
synchronized (cPage) {
if (!cPage.valid) {
- read(cPage, helper);
+ read(cPage);
cPage.valid = true;
}
}
@@ -215,9 +217,8 @@
}
pageReplacementStrategy.notifyCachePageAccess(cPage);
if(DEBUG){
- pinnedPageOwner.put((CachedPage) cPage, Thread.currentThread().getStackTrace());
+ pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace());
}
- cPage.setLargePageHelper(helper);
return cPage;
}
@@ -565,26 +566,42 @@
return false;
}
- private void read(CachedPage cPage, ILargePageHelper helper) throws HyracksDataException {
+ private void read(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
cPage.buffer.clear();
- ioManager.syncRead(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
- cPage.buffer);
- if (helper != null) {
- int totalPages = helper.getSupplementalBlockNumPages(cPage) + 1;
+ BufferCacheHeaderHelper header = checkoutHeaderHelper();
+ try {
+ long bytesRead = ioManager.syncRead(fInfo.getFileHandle(),
+ getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)), header.prepareRead());
+
+ if (bytesRead != getPageSizeWithHeader()) {
+ if (bytesRead == -1) {
+ // disk order scan code seems to rely on this behavior, so silently return
+ return;
+ }
+ throw new HyracksDataException("Failed to read a complete page: " + bytesRead);
+ }
+ int totalPages = header.processRead(cPage);
+
if (totalPages > 1) {
- resizePage(cPage, totalPages);
+ pageReplacementStrategy.fixupCapacityOnLargeRead(cPage);
cPage.buffer.position(pageSize);
cPage.buffer.limit(totalPages * pageSize);
- ioManager.syncRead(fInfo.getFileHandle(), (long) helper.getSupplementalBlockPageId(cPage) * pageSize,
- cPage.buffer);
+ ioManager.syncRead(fInfo.getFileHandle(), getOffsetForPage(cPage.getExtraBlockPageId()), cPage.buffer);
}
+ } finally {
+ returnHeaderHelper(header);
}
}
+ private long getOffsetForPage(long pageId) {
+ return pageId * getPageSizeWithHeader();
+ }
+
@Override
- public void resizePage(ICachedPage cPage, int totalPages) {
- pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages);
+ public void resizePage(ICachedPage cPage, int totalPages, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages, extraPageBlockHelper);
}
BufferedFileHandle getFileInfo(CachedPage cPage) throws HyracksDataException {
@@ -600,6 +617,17 @@
return fInfo;
}
}
+ private BufferCacheHeaderHelper checkoutHeaderHelper() {
+ BufferCacheHeaderHelper helper = headerPageCache.poll();
+ if (helper == null) {
+ helper = new BufferCacheHeaderHelper(pageSize);
+ }
+ return helper;
+ }
+
+ private void returnHeaderHelper(BufferCacheHeaderHelper buffer) {
+ headerPageCache.offer(buffer);
+ }
void write(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
@@ -607,21 +635,29 @@
synchronized (fInfo) {
if (!fInfo.fileHasBeenDeleted()) {
ByteBuffer buf = cPage.buffer.duplicate();
- buf.position(0);
- buf.limit(pageSize);
- ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
- buf);
- if (cPage.largePageHelper != null) {
- int totalPages = cPage.largePageHelper.getSupplementalBlockNumPages(cPage) + 1;
- if (totalPages > 1) {
- buf.limit(totalPages * pageSize);
- long offset = (long) cPage.largePageHelper.getSupplementalBlockPageId(cPage) * pageSize;
- ioManager.syncWrite(fInfo.getFileHandle(), offset, buf);
+ final int totalPages = cPage.getFrameSizeMultiplier();
+ final int extraBlockPageId = cPage.getExtraBlockPageId();
+ final boolean contiguousLargePages = (BufferedFileHandle.getPageId(cPage.dpid) + 1) == extraBlockPageId;
+ BufferCacheHeaderHelper header = checkoutHeaderHelper();
+ try {
+ buf.limit(contiguousLargePages ? pageSize * totalPages : pageSize);
+ buf.position(0);
+ long bytesWritten = ioManager.syncWrite(fInfo.getFileHandle(),
+ getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)),
+ header.prepareWrite(cPage, buf));
+
+ if (bytesWritten !=
+ (contiguousLargePages ? pageSize * (totalPages - 1) : 0) + getPageSizeWithHeader()) {
+ throw new HyracksDataException("Failed to write completely: " + bytesWritten);
}
- assert buf.capacity() == (pageSize * totalPages);
- } else {
- assert buf.capacity() == pageSize;
+ } finally {
+ returnHeaderHelper(header);
}
+ if (totalPages > 1 && !contiguousLargePages) {
+ buf.limit(totalPages * pageSize);
+ ioManager.syncWrite(fInfo.getFileHandle(), getOffsetForPage(extraBlockPageId), buf);
+ }
+ assert buf.capacity() == (pageSize * totalPages);
}
}
}
@@ -1134,9 +1170,9 @@
throw new HyracksDataException("No such file mapped for fileId:" + fileId);
}
if(DEBUG) {
- assert ioManager.getSize(fInfo.getFileHandle()) % getPageSize() == 0;
+ assert ioManager.getSize(fInfo.getFileHandle()) % getPageSizeWithHeader() == 0;
}
- return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSize());
+ return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSizeWithHeader());
}
}
@@ -1151,8 +1187,11 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier));
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ ICachedPage cachedPage = confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier));
+ ((ICachedPageInternal)cachedPage).setExtraBlockPageId(extraBlockPageId);
+ return cachedPage;
}
private ICachedPage confiscatePage(long dpid, Supplier<ICachedPageInternal> victimSupplier)
@@ -1236,7 +1275,6 @@
// if we found a page after all that, go ahead and finish
if (returnPage != null) {
((CachedPage) returnPage).confiscated.set(true);
- ((CachedPage) returnPage).setLargePageHelper(null);
if (DEBUG) {
confiscateLock.lock();
try{
@@ -1336,7 +1374,6 @@
}
}
pageReplacementStrategy.adviseWontNeed(cPage);
- cPage.largePageHelper = null;
}
@Override
@@ -1390,4 +1427,41 @@
}
}
+ static class BufferCacheHeaderHelper {
+ private static final int FRAME_MULTIPLIER_OFF = 0;
+ private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4
+
+ private final ByteBuffer buf;
+ private final ByteBuffer [] array;
+
+ private BufferCacheHeaderHelper(int pageSize) {
+ buf = ByteBuffer.allocate(RESERVED_HEADER_BYTES + pageSize);
+ array = new ByteBuffer[] { buf, null };
+ }
+
+ private ByteBuffer[] prepareWrite(CachedPage cPage, ByteBuffer pageBuffer) {
+ buf.position(0);
+ buf.limit(RESERVED_HEADER_BYTES);
+ buf.putInt(FRAME_MULTIPLIER_OFF, cPage.getFrameSizeMultiplier());
+ buf.putInt(EXTRA_BLOCK_PAGE_ID_OFF, cPage.getExtraBlockPageId());
+ array[1] = pageBuffer;
+ return array;
+ }
+
+ private ByteBuffer prepareRead() {
+ buf.position(0);
+ buf.limit(buf.capacity());
+ return buf;
+ }
+
+ private int processRead(CachedPage cPage) {
+ buf.position(RESERVED_HEADER_BYTES);
+ cPage.buffer.position(0);
+ cPage.buffer.put(buf);
+ int multiplier = buf.getInt(FRAME_MULTIPLIER_OFF);
+ cPage.setFrameSizeMultiplier(multiplier);
+ cPage.setExtraBlockPageId(buf.getInt(EXTRA_BLOCK_PAGE_ID_OFF));
+ return multiplier;
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
index ddab742..379d20e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
@@ -39,7 +39,11 @@
volatile boolean valid;
final AtomicBoolean confiscated;
private IQueueInfo queueInfo;
- ILargePageHelper largePageHelper;
+ private int multiplier;
+ private int extraBlockPageId;
+ // DEBUG
+ private static final boolean DEBUG = false;
+ private final StackTraceElement[] ctorStack;
//Constructor for making dummy entry for FIFO queue
public CachedPage(){
@@ -52,6 +56,7 @@
queueInfo = null;
replacementStrategyObject = null;
latch =null;
+ ctorStack = DEBUG ? new Throwable().getStackTrace() : null;
}
public CachedPage(int cpid, ByteBuffer buffer, IPageReplacementStrategy pageReplacementStrategy) {
@@ -66,6 +71,7 @@
valid = false;
confiscated = new AtomicBoolean(false);
queueInfo = null;
+ ctorStack = new Throwable().getStackTrace();
}
public void reset(long dpid) {
@@ -75,7 +81,6 @@
confiscated.set(false);
pageReplacementStrategy.notifyCachePageReset(this);
queueInfo = null;
- largePageHelper = null;
}
public void invalidate() {
@@ -154,6 +159,26 @@
return dpid;
}
+ @Override
+ public int getFrameSizeMultiplier() {
+ return multiplier;
+ }
+
+ @Override
+ public void setFrameSizeMultiplier(int multiplier) {
+ this.multiplier = multiplier;
+ }
+
+ @Override
+ public void setExtraBlockPageId(int extraBlockPageId) {
+ this.extraBlockPageId = extraBlockPageId;
+ }
+
+ @Override
+ public int getExtraBlockPageId() {
+ return extraBlockPageId;
+ }
+
CachedPage getNext() {
return next;
}
@@ -161,13 +186,4 @@
void setNext(CachedPage next) {
this.next = next;
}
-
- public ILargePageHelper getLargePageHelper() {
- return largePageHelper;
- }
-
- public void setLargePageHelper(ILargePageHelper largePageHelper) {
- this.largePageHelper = largePageHelper;
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
index ea27757..f69afa5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
@@ -25,6 +25,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
private static final Logger LOGGER = Logger.getLogger(ClockPageReplacementStrategy.class.getName());
private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3;
@@ -80,13 +82,13 @@
@Override
public ICachedPageInternal findVictim(int multiplier) {
- while (numPages.get() + (multiplier - 1) >= maxAllowedNumPages) {
+ while (numPages.get() + multiplier > maxAllowedNumPages) {
// TODO: is dropping pages on the floor enough to adhere to memory budget?
ICachedPageInternal victim = findVictimByEviction();
if (victim == null) {
return null;
}
- int multiple = victim.getBuffer().capacity() / pageSize;
+ int multiple = victim.getFrameSizeMultiplier();
if (multiple == multiplier) {
return victim;
} else if (bufferCache.removePage(victim)) {
@@ -153,28 +155,33 @@
cpId = cpIdCounter.getAndIncrement();
}
CachedPage cPage = new CachedPage(cpId, allocator.allocate(pageSize * multiplier, 1)[0], this);
+ cPage.setFrameSizeMultiplier(multiplier);
bufferCache.addPage(cPage);
numPages.getAndAdd(multiplier);
return cPage;
}
@Override
- public void resizePage(ICachedPageInternal cPage, int multiplier) {
- ByteBuffer oldBuffer = cPage.getBuffer();
- int origMultiplier = oldBuffer.capacity() / pageSize;
+ public void resizePage(ICachedPageInternal cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ int origMultiplier = cPage.getFrameSizeMultiplier();
if (origMultiplier == multiplier) {
// no-op
return;
}
final int newSize = pageSize * multiplier;
+ ByteBuffer oldBuffer = ((CachedPage)cPage).buffer;
oldBuffer.position(0);
if (multiplier < origMultiplier) {
oldBuffer.limit(newSize);
+ final int gap = origMultiplier - multiplier;
+ // we return the unused portion of our block to the page manager
+ extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId() + gap, gap);
} else {
while (numPages.get() + (multiplier - origMultiplier) >= maxAllowedNumPages) {
ICachedPageInternal victim = findVictimByEviction();
if (victim != null) {
- int multiple = victim.getBuffer().capacity() / pageSize;
+ int multiple = victim.getFrameSizeMultiplier();
if (bufferCache.removePage(victim)) {
// TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
cpIdFreeList.add(victim.getCachedPageId());
@@ -192,10 +199,51 @@
}
}
+ if (origMultiplier != 1) {
+ // return the old block to the page manager
+ extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId(), origMultiplier);
+ }
+ cPage.setExtraBlockPageId(extraPageBlockHelper.getFreeBlock(multiplier));
}
+ cPage.setFrameSizeMultiplier(multiplier);
ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0];
newBuffer.put(oldBuffer);
numPages.getAndAdd(multiplier - origMultiplier);
+ ((CachedPage) cPage).buffer = newBuffer;
+ }
+
+ @Override
+ public void fixupCapacityOnLargeRead(ICachedPageInternal cPage)
+ throws HyracksDataException {
+ ByteBuffer oldBuffer = ((CachedPage) cPage).buffer;
+ final int multiplier = cPage.getFrameSizeMultiplier();
+ final int newSize = pageSize * multiplier;
+ final int delta = multiplier - 1;
+ oldBuffer.position(0);
+ while (numPages.get() + delta >= maxAllowedNumPages) {
+ ICachedPageInternal victim = findVictimByEviction();
+ if (victim != null) {
+ final int victimMultiplier = victim.getFrameSizeMultiplier();
+ if (bufferCache.removePage(victim)) {
+ // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
+ cpIdFreeList.add(victim.getCachedPageId());
+ numPages.getAndAdd(-victimMultiplier);
+ }
+ } else {
+ // TODO(mblow): what should we do with we need to resize and don't have the budget?
+ // we don't have the budget to resize- proceed anyway, but log
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by "
+ + (numPages.get() + delta - maxAllowedNumPages)
+ + " pages in order to satisfy large page read");
+ }
+ break;
+
+ }
+ }
+ ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0];
+ newBuffer.put(oldBuffer);
+ numPages.getAndAdd(delta);
((CachedPage) cPage).buffer = newBuffer;
}
@@ -232,4 +280,5 @@
//make the page appear as if it wasn't accessed even if it was
getPerPageObject(cPage).set(false);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index c78012b..4a41ec0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -88,13 +88,6 @@
}
@Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(dpid, newPage, helper);
- pinCount.addAndGet(1);
- return page;
- }
-
- @Override
public void unpin(ICachedPage page) throws HyracksDataException {
bufferCache.unpin(page);
unpinCount.addAndGet(1);
@@ -204,8 +197,9 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return bufferCache.confiscateLargePage(dpid, multiplier);
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ return bufferCache.confiscateLargePage(dpid, multiplier, extraBlockPageId);
}
@Override
@@ -262,7 +256,8 @@
}
@Override
- public void resizePage(ICachedPage page, int multiple) {
- bufferCache.resizePage(page, multiple);
+ public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ bufferCache.resizePage(page, multiplier, extraPageBlockHelper);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index feac69e..ca87673 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -24,8 +24,9 @@
public interface IBufferCache {
- public static final long INVALID_DPID = -1l;
- public static final int INVALID_PAGEID = -1;
+ long INVALID_DPID = -1l;
+ int INVALID_PAGEID = -1;
+ int RESERVED_HEADER_BYTES = 8;
public void createFile(FileReference fileRef) throws HyracksDataException;
@@ -43,8 +44,6 @@
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException;
-
public void unpin(ICachedPage page) throws HyracksDataException;
public void flushDirtyPage(ICachedPage page) throws HyracksDataException;
@@ -53,7 +52,7 @@
public ICachedPage confiscatePage(long dpid) throws HyracksDataException;
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException;
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException;
public void returnPage(ICachedPage page);
@@ -61,7 +60,7 @@
public void force(int fileId, boolean metadata) throws HyracksDataException;
- public int getPageSize();
+ int getPageSize();
public int getNumPages();
@@ -85,5 +84,6 @@
void purgeHandle(int fileId) throws HyracksDataException;
- void resizePage(ICachedPage page, int multiple);
+ void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
index 12dde49..8107234 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
@@ -21,19 +21,22 @@
import java.nio.ByteBuffer;
public interface ICachedPage {
- public ByteBuffer getBuffer();
- public void acquireReadLatch();
+ ByteBuffer getBuffer();
- public void releaseReadLatch();
+ void acquireReadLatch();
- public void acquireWriteLatch();
+ void releaseReadLatch();
- public void releaseWriteLatch(boolean markDirty);
+ void acquireWriteLatch();
- public boolean confiscated();
+ void releaseWriteLatch(boolean markDirty);
- public IQueueInfo getQueueInfo();
+ boolean confiscated();
- public void setQueueInfo(IQueueInfo queueInfo);
+ IQueueInfo getQueueInfo();
+
+ void setQueueInfo(IQueueInfo queueInfo);
+
+ int getFrameSizeMultiplier();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
index 6a6c2f5..c500286 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
@@ -27,4 +27,9 @@
public boolean isGoodVictim();
+ void setFrameSizeMultiplier(int multiplier);
+
+ int getExtraBlockPageId();
+
+ void setExtraBlockPageId(int extraBlockPageId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
similarity index 77%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
index 22bec97..ad7f2f6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
@@ -18,8 +18,9 @@
*/
package org.apache.hyracks.storage.common.buffercache;
-public interface ILargePageHelper {
+import org.apache.hyracks.api.exceptions.HyracksDataException;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
+public interface IExtraPageBlockHelper {
+ int getFreeBlock(int size) throws HyracksDataException;
+ void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
index 1a59a65..d3bcce5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.storage.common.buffercache;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public interface IPageReplacementStrategy {
public Object createPerPageStrategyObject(int cpid);
@@ -37,9 +39,13 @@
public int getNumPages();
+ void fixupCapacityOnLargeRead(ICachedPageInternal cPage)
+ throws HyracksDataException;
+
public int getPageSize();
public int getMaxAllowedNumPages();
- public void resizePage(ICachedPageInternal page, int multiplier);
+ void resizePage(ICachedPageInternal page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index 840acd5..4420fb0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -23,7 +23,6 @@
import java.util.Random;
import java.util.logging.Level;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.junit.Assert;
import org.junit.Test;
@@ -136,8 +135,7 @@
int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
bufferCache.openFile(btreeFileId);
IHyracksTaskContext ctx = harness.getHyracksTaskContext();
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true);
try {
ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
index cce2f82..01863ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
@@ -24,7 +24,6 @@
import java.util.Random;
import java.util.logging.Level;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.junit.Test;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -90,8 +89,7 @@
}
try {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
LatchType latch = null;
switch (fta) {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
index 2330e4d..e85ff9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
@@ -18,15 +18,9 @@
*/
package org.apache.hyracks.storage.common;
-import static org.junit.Assert.fail;
-
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.io.FileReference;
@@ -34,12 +28,18 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IIOManager.FileReadWriteMode;
import org.apache.hyracks.api.io.IIOManager.FileSyncMode;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
public class BufferCacheRegressionTest {
protected static final String tmpDir = System.getProperty("java.io.tmpdir");
@@ -126,9 +126,9 @@
FileReference testFileRef = new FileReference(new File(fileName));
IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY,
FileSyncMode.METADATA_SYNC_DATA_SYNC);
- ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE);
+ ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE + BufferCache.RESERVED_HEADER_BYTES);
ioManager.syncRead(testFileHandle, 0, testBuffer);
- for (int i = 0; i < testBuffer.capacity(); i++) {
+ for (int i = BufferCache.RESERVED_HEADER_BYTES; i < testBuffer.capacity(); i++) {
if (deleteFile) {
// We deleted the file. We expect to see a clean buffer.
if (testBuffer.get(i) == Byte.MAX_VALUE) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 3:
(2 comments)
https://asterix-gerrit.ics.uci.edu/#/c/920/3/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
File hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java:
Line 41: public boolean isLargePage();
The naming conventions between the last 4 methods seem a little inconsistent.
https://asterix-gerrit.ics.uci.edu/#/c/920/3/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
File hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java:
Line 74: ctorStack = new Throwable().getStackTrace();
Should this stay in?
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/920
to look at the new patch set (#3).
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj
Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart
- large page information managed by buffer cache, outside of page
- size header written out a prefix to pages
Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
54 files changed, 492 insertions(+), 439 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/920/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 3:
(2 comments)
https://asterix-gerrit.ics.uci.edu/#/c/920/3/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
File hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java:
Line 41: public boolean isLargePage();
> The naming conventions between the last 4 methods seem a little inconsisten
Done
https://asterix-gerrit.ics.uci.edu/#/c/920/3/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
File hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java:
Line 74: ctorStack = new Throwable().getStackTrace();
> Should this stay in?
Good catch- I missed this when adding the DEBUG guard. Will address.
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 4:
WARNING: THIS CHANGE CONTAINS CROSS-PRODUCT CHANGES IN:
* asterixdb
* hyracks-fullstack
PLEASE REVIEW CAREFULLY AND LOOK FOR API CHANGES!
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 2:
(6 comments)
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
File hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java:
Line 46: public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
> looking at this method definition, no one would guess that the dataArray is
IIOManager is generic and can be used for lots of things, I don't think we would want to corrupt this interface with BufferCache's specific use pattern
Line 50: public long syncRead(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
> Remove unused? or comment future use?
Done
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
File hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java:
Line 189: @Override
> remove method?
Done
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
File hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java:
Line 47:
> Am I seeing tabs??
Yes, I think we should remove tabs from files in a separate change (they're not being introduced here)
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
File hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java:
Line 46: ITreeIndexFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame)
> Huh! how come Sonar doesn't detect those?
Because they are not new.
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
File hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java:
Line 228: // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
> I believe it is enough. would be nice to find a better source to confirm
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 3:
WARNING: THIS CHANGE CONTAINS CROSS-PRODUCT CHANGES IN:
* asterixdb
* hyracks-fullstack
PLEASE REVIEW CAREFULLY AND LOOK FOR API CHANGES!
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 1:
WARNING: THIS CHANGE CONTAINS CROSS-PRODUCT CHANGES IN:
* asterixdb
* hyracks-fullstack
PLEASE REVIEW CAREFULLY AND LOOK FOR API CHANGES!
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 3:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1646/
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has submitted this change and it was merged.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj
Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart
- large page information managed by buffer cache, outside of page
- size header written out a prefix to pages
Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/920
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
54 files changed, 496 insertions(+), 447 deletions(-)
Approvals:
abdullah alamoudi: Looks good to me, approved
Jenkins: Verified
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
index 7d92ce2..9fb4e94 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.common.config;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
public class AsterixStorageProperties extends AbstractAsterixProperties {
private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize";
@@ -62,7 +64,7 @@
}
public int getBufferCacheNumPages() {
- return (int) (getBufferCacheSize() / getBufferCachePageSize());
+ return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
}
public int getBufferCacheMaxOpenFiles() {
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
similarity index 66%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
index 22bec97..77c8179 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
@@ -16,10 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+/*
+ * Test case Name : big_object_bulkload.aql
+ * Description : bulkload insert of large objects
+ * Expected Result : Success
+ * Date : 20th April 2016
+ */
-public interface ILargePageHelper {
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use dataverse testdv2;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
+create type testtype as closed {
+ id: int64,
+ name: string,
+ hobbies: {{string}}
}
+
+create dataset testds(testtype) primary key id;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
similarity index 74%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
index 22bec97..f1b3819 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
@@ -16,10 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+/**
+ *
+ * Big object (20 MB) loading test
+ * Expected result: success
+ *
+ */
-public interface ILargePageHelper {
+use dataverse testdv2;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
-}
+load dataset testds
+using localfs
+(("path"="asterix_nc1://../../../../../../asterix-app/target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
similarity index 79%
rename from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
rename to asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
index 22bec97..23c1bed 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
@@ -16,10 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.storage.common.buffercache;
+use dataverse testdv2;
-public interface ILargePageHelper {
-
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
-}
+for $d in dataset("testds")
+where $d.id = 1
+return $d
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
new file mode 100644
index 0000000..5acf078
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
@@ -0,0 +1 @@
+{ "id": 1, "name": "Person One", "hobbies": {{ "%lorembytes:20971520%" }} }
\ No newline at end of file
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
new file mode 100755
index 0000000..37bc7b0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
@@ -0,0 +1,2 @@
+jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9;
+$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
new file mode 100755
index 0000000..5f247db
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix delete -n asterix;
+
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
new file mode 100755
index 0000000..e2b0583
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
@@ -0,0 +1,2 @@
+$MANAGIX_HOME/bin/managix stop -n asterix;
+$MANAGIX_HOME/bin/managix start -n asterix;
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index 08b8e87..75b2457 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -28,6 +28,11 @@
<output-dir compare="Text">external_index</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="query_after_restart">
+ <compilation-unit name="big_object_20M">
+ <output-dir compare="Text">big_object_20M</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="recover_after_abort">
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index 47eac7e..63709ed 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -43,6 +43,8 @@
public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
+ public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
+
public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException;
public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index 519a9d7..6588f3b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -22,6 +22,7 @@
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -49,7 +50,7 @@
public IOManager(List<IODeviceHandle> devices) throws HyracksException {
this.ioDevices = Collections.unmodifiableList(devices);
- workAreaIODevices = new ArrayList<IODeviceHandle>();
+ workAreaIODevices = new ArrayList<>();
for (IODeviceHandle d : ioDevices) {
if (d.getWorkAreaPath() != null) {
new File(d.getPath(), d.getWorkAreaPath()).mkdirs();
@@ -109,13 +110,48 @@
}
}
+ @Override
+ public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException {
+ try {
+ if (fHandle == null) {
+ throw new IllegalStateException("Trying to write to a deleted file.");
+ }
+ int n = 0;
+ int remaining = 0;
+ for (ByteBuffer buf : dataArray) {
+ remaining += buf.remaining();
+ }
+ final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel();
+ while (remaining > 0) {
+ long len;
+ synchronized (fileChannel) {
+ fileChannel.position(offset);
+ len = fileChannel.write(dataArray);
+ }
+ if (len < 0) {
+ throw new HyracksDataException(
+ "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString());
+ }
+ remaining -= len;
+ offset += len;
+ n += len;
+ }
+ return n;
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
/**
* Please do check the return value of this read!
*
* @param fHandle
* @param offset
* @param data
- * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's current size
+ * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's
+ * current size
* @throws HyracksDataException
*/
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
index 1081dc8..e0abb1c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
@@ -38,5 +38,7 @@
public void setLargeFlag(boolean largePage);
+ public boolean getLargeFlag();
+
public void validate(PageValidationInfo pvi) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index 7648ee6..2e072ce 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -21,14 +21,13 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode;
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public interface IBTreeLeafFrame extends IBTreeFrame {
public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp,
@@ -53,12 +52,6 @@
public int getNextLeaf();
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId);
-
- public boolean isLargePage();
-
- public int getSupplementalNumPages();
-
- void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
- IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException;
+ void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index 701f44f..258a5d6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -33,11 +33,9 @@
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixPrefixTupleReference;
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import org.apache.hyracks.storage.am.btree.impls.FieldPrefixTupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameCompressor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -51,7 +49,7 @@
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
/**
* WARNING: only works when tupleWriter is an instance of TypeAwareTupleWriter
@@ -537,23 +535,13 @@
}
@Override
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId) {
- throw new IllegalStateException();
- }
-
- @Override
public void setLargeFlag(boolean largePage) {
throw new IllegalStateException();
}
@Override
- public boolean isLargePage() {
+ public boolean getLargeFlag() {
return false;
- }
-
- @Override
- public int getSupplementalNumPages() {
- throw new IllegalStateException();
}
public int getPrefixTupleCount() {
@@ -596,7 +584,8 @@
}
@Override
- public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException {
BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame;
@@ -804,12 +793,8 @@
}
@Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
-
- @Override
- public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException {
+ public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper helper)
+ throws HyracksDataException {
throw new IllegalStateException("nyi");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
index c558d30..f7b270c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
deleted file mode 100644
index 79f7f53..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.btree.frames;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.storage.common.buffercache.CachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
-
-public class BTreeLargeFrameHelper implements ILargePageHelper {
-
- public static final BTreeLargeFrameHelper INSTANCE = new BTreeLargeFrameHelper();
-
- private BTreeLargeFrameHelper() {
- }
-
- @Override
- public int getSupplementalBlockNumPages(CachedPage cPage) {
- ByteBuffer buffer = cPage.getBuffer();
- return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalNumPages(buffer) : 0;
- }
-
- @Override
- public int getSupplementalBlockPageId(CachedPage cPage) {
- ByteBuffer buffer = cPage.getBuffer();
- return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalPageId(buffer) : -1;
- }
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index c2b18b7..9505f8a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -29,10 +29,8 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -43,6 +41,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame {
@@ -55,7 +54,7 @@
private MultiComparator cmp;
public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
- super(tupleWriter, new OrderedSlotManager(), null);
+ super(tupleWriter, new OrderedSlotManager());
cmpFrameTuple = tupleWriter.createTupleReference();
previousFt = tupleWriter.createTupleReference();
}
@@ -201,7 +200,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException, TreeIndexException {
ByteBuffer right = rightFrame.getBuffer();
int tupleCount = getTupleCount();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
index 3737486..4ecb83d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 8f560fe..406b494 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -25,10 +25,8 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -40,25 +38,23 @@
import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
protected static final int nextLeafOff = flagOff + 1; // 22
- protected static final int supplementalNumPagesOff = nextLeafOff + 4; // 26
- protected static final int supplementalPageIdOff = supplementalNumPagesOff + 4; // 30
private MultiComparator cmp;
private final ITreeIndexTupleReference previousFt;
- public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, ILargePageHelper largePageHelper) {
- super(tupleWriter, new OrderedSlotManager(), largePageHelper);
+ public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
+ super(tupleWriter, new OrderedSlotManager());
previousFt = tupleWriter.createTupleReference();
}
@Override
public int getPageHeaderSize() {
- return supplementalPageIdOff + 4;
+ return nextLeafOff + 4;
}
@Override
@@ -70,8 +66,6 @@
public void initBuffer(byte level) {
super.initBuffer(level);
buf.putInt(nextLeafOff, -1);
- buf.putInt(supplementalNumPagesOff, 0);
- buf.putInt(supplementalPageIdOff, -1);
}
@Override
@@ -82,28 +76,6 @@
@Override
public int getNextLeaf() {
return buf.getInt(nextLeafOff);
- }
-
- public static int getSupplementalNumPages(ByteBuffer buf) {
- return buf.getInt(supplementalNumPagesOff);
- }
-
- public int getSupplementalNumPages() {
- return getSupplementalNumPages(buf);
- }
-
- public static int getSupplementalPageId(ByteBuffer buf) {
- return buf.getInt(supplementalPageIdOff);
- }
-
- public int getSupplementalPageId() {
- return getSupplementalPageId(buf);
- }
-
- public void configureLargePage(int supplementalPages, int supplementalBlockPageId) {
- setLargeFlag(true);
- buf.putInt(supplementalNumPagesOff, supplementalPages);
- buf.putInt(supplementalPageIdOff, supplementalBlockPageId);
}
@Override
@@ -204,7 +176,7 @@
boolean isLargeTuple(int tupleSize) {
// TODO(mblow): make page size available to avoid calculating it
- int pageSize = isLargePage() ? buf.capacity() / (getSupplementalNumPages() + 1) : buf.capacity();
+ int pageSize = getLargeFlag() ? buf.capacity() / page.getFrameSizeMultiplier() : buf.capacity();
return tupleSize > getMaxTupleSize(pageSize);
}
@@ -228,7 +200,7 @@
int oldTupleBytes = frameTuple.getTupleSize();
int newTupleBytes = tupleWriter.bytesRequired(newTuple);
FrameOpSpaceStatus status = hasSpaceUpdate(oldTupleBytes, newTupleBytes);
- if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (isLargePage() || getTupleCount() == 1)
+ if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (getLargeFlag() || getTupleCount() == 1)
&& isLargeTuple(newTupleBytes)) {
return FrameOpSpaceStatus.EXPAND;
}
@@ -237,8 +209,8 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
- throws HyracksDataException {
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
+ throws HyracksDataException {
int tupleSize = getBytesRequiredToWriteTuple(tuple);
@@ -249,7 +221,7 @@
// Find split point, and determine into which frame the new tuple should
// be inserted into.
- BTreeNSMLeafFrame targetFrame = null;
+ BTreeNSMLeafFrame targetFrame;
frameTuple.resetByTupleIndex(this, tupleCount - 1);
if (cmp.compare(tuple, frameTuple) > 0) {
// This is a special optimization case when the tuple to be inserted is the largest key on the page.
@@ -275,22 +247,14 @@
targetFrame = this;
}
int tuplesToRight = tupleCount - tuplesToLeft;
- int supplementalPages = 0;
- int supplementalPageId = -1;
- if (isLargePage()) {
- ((BTreeNSMLeafFrame) rightFrame).growCapacity(freePageManager, metaFrame, bufferCache,
+ if (getLargeFlag()) {
+ ((BTreeNSMLeafFrame) rightFrame).growCapacity(extraPageBlockHelper, bufferCache,
buf.capacity() - rightFrame.getBuffer().capacity());
- supplementalPages = ((BTreeNSMLeafFrame) rightFrame).getSupplementalNumPages();
- supplementalPageId = ((BTreeNSMLeafFrame) rightFrame).getSupplementalPageId();
}
ByteBuffer right = rightFrame.getBuffer();
// Copy entire page.
System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity());
- if (isLargePage()) {
- // restore the supplemental page metadata
- ((BTreeNSMLeafFrame) rightFrame).configureLargePage(supplementalPages, supplementalPageId);
- }
// On the right page we need to copy rightmost slots to the left.
int src = rightFrame.getSlotManager().getSlotEndOff();
@@ -309,7 +273,7 @@
}
if (tupleLarge) {
- targetFrame.ensureCapacity(freePageManager, metaFrame, bufferCache, tuple);
+ targetFrame.ensureCapacity(bufferCache, tuple, extraPageBlockHelper);
}
// Insert the new tuple.
@@ -317,7 +281,7 @@
// it's safe to catch this exception since it will have been caught
// before reaching here
try {
- targetTupleIndex = ((BTreeNSMLeafFrame) targetFrame).findInsertTupleIndex(tuple);
+ targetTupleIndex = targetFrame.findInsertTupleIndex(tuple);
} catch (TreeIndexException e) {
throw new IllegalStateException(e);
}
@@ -332,13 +296,13 @@
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
}
- public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
- IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException {
+ public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple,
+ IExtraPageBlockHelper extraPageBlockHelper) throws HyracksDataException {
int gapBytes = getBytesRequiredToWriteTuple(tuple) - getFreeContiguousSpace();
- growCapacity(freePageManager, metaFrame, bufferCache, gapBytes);
+ growCapacity(extraPageBlockHelper, bufferCache, gapBytes);
}
- public void growCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame,
+ public void growCapacity(IExtraPageBlockHelper extraPageBlockHelper,
IBufferCache bufferCache, int delta) throws HyracksDataException {
if (delta <= 0) {
setLargeFlag(true);
@@ -346,35 +310,21 @@
}
int deltaPages = (int) Math.ceil((double) delta / bufferCache.getPageSize());
int framePagesOld = getBuffer().capacity() / bufferCache.getPageSize();
- int oldSupplementalPages = 0;
- int oldSupplementalPageId = -1;
- if (isLargePage()) {
- oldSupplementalPages = getSupplementalNumPages();
- oldSupplementalPageId = getSupplementalPageId();
- }
-
- configureLargePage(framePagesOld + deltaPages - 1,
- freePageManager.getFreePageBlock(metaFrame, framePagesOld + deltaPages - 1));
-
- int pageDelta = (framePagesOld + deltaPages) - 1 - oldSupplementalPages;
+ int newMultiplier = framePagesOld + deltaPages;
// we need to get the old slot offsets before we grow
int oldSlotEnd = slotManager.getSlotEndOff();
int oldSlotStart = slotManager.getSlotStartOff() + slotManager.getSlotSize();
- bufferCache.resizePage(getPage(), framePagesOld + deltaPages);
- buf = getPage().getBuffer();
+ bufferCache.resizePage(getPage(), newMultiplier, extraPageBlockHelper);
- // return the dropped supplemental pages to the page manager...
- if (oldSupplementalPages > 0) {
- freePageManager.addFreePageBlock(metaFrame, oldSupplementalPageId, oldSupplementalPages);
- }
+ buf = getPage().getBuffer();
// fixup the slots
System.arraycopy(buf.array(), oldSlotEnd, buf.array(), slotManager.getSlotEndOff(), oldSlotStart - oldSlotEnd);
// fixup total free space counter
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * pageDelta));
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * deltaPages));
}
@Override
@@ -416,8 +366,6 @@
public String printHeader() {
StringBuilder strBuilder = new StringBuilder(super.printHeader());
strBuilder.append("nextLeafOff: " + nextLeafOff + "\n");
- strBuilder.append("supplementalNumPagesOff: " + supplementalNumPagesOff + "\n");
- strBuilder.append("supplementalPageIdOff: " + supplementalPageIdOff + "\n");
return strBuilder.toString();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
index 2b7f12b..3cf0ba5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
@@ -22,7 +22,6 @@
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -36,7 +35,7 @@
@Override
public IBTreeLeafFrame createFrame() {
- return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), getLargePageHelper());
+ return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
@Override
@@ -44,8 +43,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return BTreeLargeFrameHelper.INSTANCE;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 3043940..5995004 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -66,7 +66,6 @@
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.apache.hyracks.storage.common.buffercache.CachedPage;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
@@ -103,8 +102,7 @@
RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp);
int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
int currentPageId = bulkloadLeafStart;
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false,
- largePageHelper);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
try {
cursor.setBufferCache(bufferCache);
@@ -136,7 +134,7 @@
}
private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(page);
PageValidationInfo currentPvi = ctx.validationInfos.peekFirst();
@@ -220,8 +218,7 @@
ICachedPage originalPage = ctx.interiorFrame.getPage();
for (int i = 0; i < ctx.smPages.size(); i++) {
int pageId = ctx.smPages.get(i);
- ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false,
- largePageHelper);
+ ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
smPage.acquireWriteLatch();
try {
ctx.interiorFrame.setPage(smPage);
@@ -246,17 +243,17 @@
private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
// Make sure the root is always in the same page.
ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()),
- false, largePageHelper);
+ false);
leftNode.acquireWriteLatch();
try {
int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true,
- largePageHelper);
+ ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
newLeftNode.acquireWriteLatch();
try {
boolean largePage = false;
if (leftNode.getBuffer().capacity() > newLeftNode.getBuffer().capacity()) {
- bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize());
+ bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize(),
+ ctx);
largePage = true;
}
// Copy left child to new left child.
@@ -268,7 +265,7 @@
long leftNodeLSN = ctx.interiorFrame.getPageLsn();
// Initialize new root (leftNode becomes new root).
if (largePage) {
- bufferCache.resizePage(leftNode, 1);
+ bufferCache.resizePage(leftNode, 1, ctx);
ctx.interiorFrame.setPage(leftNode);
ctx.interiorFrame.setLargeFlag(false);
} else {
@@ -372,7 +369,7 @@
switch (spaceStatus) {
case EXPAND: {
// TODO: avoid repeated calculation of tuple size
- ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple);
+ ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx);
}
// fall-through
case SUFFICIENT_CONTIGUOUS_SPACE: {
@@ -432,8 +429,7 @@
}
}
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true,
- largePageHelper);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
@@ -449,7 +445,7 @@
} else {
ctx.modificationCallback.found(null, tuple);
}
- ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, bufferCache);
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache);
ctx.smPages.add(pageId);
ctx.smPages.add(rightPageId);
@@ -492,7 +488,7 @@
ctx.modificationCallback.found(beforeTuple, tuple);
ctx.leafFrame.delete(tuple, oldTupleIndex);
ctx.leafFrame.compact();
- ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple);
+ ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx);
int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
ctx.leafFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
@@ -550,8 +546,7 @@
switch (spaceStatus) {
case INSUFFICIENT_SPACE: {
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true,
- largePageHelper);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
rightNode.acquireWriteLatch();
try {
IBTreeFrame rightFrame = ctx.createInteriorFrame();
@@ -560,8 +555,7 @@
rightFrame.setMultiComparator(ctx.cmp);
// instead of creating a new split key, use the existing
// splitKey
- ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, freePageManager,
- ctx.metaFrame, bufferCache);
+ ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, ctx, bufferCache);
ctx.smPages.add(pageId);
ctx.smPages.add(rightPageId);
ctx.interiorFrame.setSmFlag(true);
@@ -631,7 +625,7 @@
}
private ICachedPage isConsistent(int pageId, BTreeOpContext ctx) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
ctx.interiorFrame.setPage(node);
boolean isConsistent = ctx.pageLsns.getLast() == ctx.interiorFrame.getPageLsn();
@@ -645,7 +639,7 @@
private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx)
throws HyracksDataException, TreeIndexException {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
ctx.interiorFrame.setPage(node);
// this check performs an unprotected read in the page
// the following could happen: TODO fill out
@@ -707,7 +701,7 @@
// Is there a propagated split key?
if (ctx.splitKey.getBuffer() != null) {
ICachedPage interiorNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ BufferedFileHandle.getDiskPageId(fileId, pageId), false);
interiorNode.acquireWriteLatch();
try {
// Insert or update op. Both can cause split keys to propagate upwards.
@@ -839,8 +833,7 @@
private BTreeOpContext createOpContext(IIndexAccessor accessor, IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new BTreeOpContext(accessor, leafFrameFactory, interiorFrameFactory,
- freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback,
- searchCallback);
+ freePageManager, cmpFactories, modificationCallback, searchCallback);
}
@SuppressWarnings("rawtypes")
@@ -857,7 +850,7 @@
public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes,
StringBuilder strBuilder, MultiComparator cmp) throws Exception {
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper);
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
try {
if (parent != null && unpin == true) {
@@ -1077,21 +1070,18 @@
int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize());
final int multiplier = (int) Math
.ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize));
- leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier);
- ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper);
+ if (multiplier > 1) {
+ leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier,
+ freePageManager.getFreePageBlock(metaFrame, multiplier - 1));
+ } else {
+ leafFrontier.page = bufferCache.confiscatePage(dpid);
+ }
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
- if (multiplier > 1) {
- int supplementalPages = multiplier - 1;
- ((IBTreeLeafFrame) leafFrame).configureLargePage(supplementalPages,
- freePageManager.getFreePageBlock(metaFrame, supplementalPages));
- } else {
- ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
- }
+ ((IBTreeLeafFrame) leafFrame).setLargeFlag(true);
} else {
final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId);
leafFrontier.page = bufferCache.confiscatePage(dpid);
- ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper);
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
}
@@ -1161,7 +1151,6 @@
propagateBulk(level + 1, pagesToWrite);
frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID);
- ((CachedPage) frontier.page).setLargePageHelper(largePageHelper);
interiorFrame.setPage(frontier.page);
interiorFrame.initBuffer((byte) level);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
index 974860e..2297cc5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.apache.hyracks.storage.am.common.api.ICursorInitialState;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
@@ -119,8 +118,7 @@
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
do {
- ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
if (exclusiveLatchNodes) {
nextLeaf.acquireWriteLatch();
page.releaseWriteLatch(isPageDirty);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
index 8513368..3649433 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
@@ -33,6 +33,7 @@
import org.apache.hyracks.storage.am.btree.api.ITupleAcceptor;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
@@ -43,8 +44,9 @@
import org.apache.hyracks.storage.am.common.ophelpers.IntArrayList;
import org.apache.hyracks.storage.am.common.ophelpers.LongArrayList;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
-public class BTreeOpContext implements IIndexOperationContext {
+public class BTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper {
private final int INIT_ARRAYLIST_SIZE = 6;
public IIndexAccessor accessor;
@@ -53,7 +55,8 @@
public ITreeIndexFrameFactory interiorFrameFactory;
public IBTreeLeafFrame leafFrame;
public IBTreeInteriorFrame interiorFrame;
- public ITreeIndexMetaDataFrame metaFrame;
+ public final IMetaDataPageManager freePageManager;
+ public final ITreeIndexMetaDataFrame metaFrame;
public IndexOperation op;
public ITreeIndexCursor cursor;
public BTreeCursorInitialState cursorInitialState;
@@ -75,9 +78,9 @@
public final ITreeIndexTupleReference leafFrameTuple;
public BTreeOpContext(IIndexAccessor accessor, ITreeIndexFrameFactory leafFrameFactory,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexMetaDataFrame metaFrame,
- IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback,
- ISearchOperationCallback searchCallback) {
+ ITreeIndexFrameFactory interiorFrameFactory, IMetaDataPageManager freePageManager,
+ IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback,
+ ISearchOperationCallback searchCallback) {
this.accessor = accessor;
if (cmpFactories[0] != null) {
@@ -97,7 +100,8 @@
if (interiorFrame != null && this.cmp != null) {
interiorFrame.setMultiComparator(cmp);
}
- this.metaFrame = metaFrame;
+ this.freePageManager = freePageManager;
+ this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.pageLsns = new LongArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
this.smoCount = 0;
this.modificationCallback = modificationCallback;
@@ -233,4 +237,14 @@
this.modificationCallback = modificationCallback;
this.searchCallback = searchCallback;
}
+
+ @Override
+ public int getFreeBlock(int size) throws HyracksDataException {
+ return freePageManager.getFreePageBlock(metaFrame, size);
+ }
+
+ @Override
+ public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException {
+ freePageManager.addFreePageBlock(metaFrame, blockPageId, size);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 975bd9b..3301e37 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -25,7 +25,6 @@
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.dataflow.common.util.TupleUtils;
import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.apache.hyracks.storage.am.common.api.ICursorInitialState;
import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
@@ -118,8 +117,7 @@
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
do {
- ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
if (exclusiveLatchNodes) {
nextLeaf.acquireWriteLatch();
page.releaseWriteLatch(isPageDirty);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
index 5b6cee9..7c2abb1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -38,7 +38,6 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeUtils {
@@ -49,7 +48,6 @@
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper();
IMetaDataPageManager freePageManager;
freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
@@ -63,7 +61,6 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper();
BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
cmpFactories, typeTraits.length, file);
return btree;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
index 711db9e..037c183 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -27,7 +27,7 @@
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public interface ITreeIndexFrame {
@@ -73,7 +73,7 @@
public String printHeader();
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException, TreeIndexException;
public ISlotManager getSlotManager();
@@ -104,6 +104,4 @@
public ITreeIndexTupleReference createTupleReference();
public void setMultiComparator(MultiComparator cmp);
-
- ILargePageHelper getLargePageHelper();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
index 248baf5..8a705dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
@@ -20,10 +20,7 @@
import java.io.Serializable;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
-
public interface ITreeIndexFrameFactory extends Serializable {
ITreeIndexFrame createFrame();
ITreeIndexTupleWriterFactory getTupleWriterFactory();
- ILargePageHelper getLargePageHelper();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
index ea6bcac..72c6fe2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
@@ -31,7 +31,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
@@ -51,14 +50,12 @@
protected ITreeIndexTupleWriter tupleWriter;
protected ITreeIndexTupleReference frameTuple;
- protected ILargePageHelper largePageHelper;
- public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager, ILargePageHelper largePageHelper) {
+ public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) {
this.tupleWriter = tupleWriter;
this.frameTuple = tupleWriter.createTupleReference();
this.slotManager = slotManager;
this.slotManager.setFrame(this);
- this.largePageHelper = largePageHelper;
}
@Override
@@ -101,12 +98,8 @@
}
}
- public static boolean isLargePage(ByteBuffer buf) {
+ public boolean getLargeFlag() {
return (buf.get(flagOff) & largeFlagBit) != 0;
- }
-
- public boolean isLargePage() {
- return isLargePage(buf);
}
@Override
@@ -342,8 +335,4 @@
return buf.capacity() - getFreeSpaceOff() - (getTupleCount() * slotManager.getSlotSize());
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return largePageHelper;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index b7b39d4..3a7eb77 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -26,12 +26,20 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import org.apache.hyracks.storage.am.common.api.IndexException;
+import org.apache.hyracks.storage.am.common.api.TreeIndexException;
import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -62,8 +70,6 @@
protected int bulkloadLeafStart = 0;
- protected final ILargePageHelper largePageHelper;
-
public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
IMetaDataPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
@@ -78,7 +84,6 @@
this.cmpFactories = cmpFactories;
this.fieldCount = fieldCount;
this.file = file;
- this.largePageHelper = leafFrameFactory.getLargePageHelper();
}
public synchronized void create() throws HyracksDataException {
@@ -125,8 +130,7 @@
ITreeIndexFrame frame = leafFrameFactory.createFrame();
ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
freePageManager.init(metaFrame, rootPage);
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
rootNode.acquireWriteLatch();
try {
frame.setPage(rootNode);
@@ -244,8 +248,7 @@
if(freePageManager.appendOnlyMode() && bufferCache.getNumPagesOfFile(fileId) <= MINIMAL_TREE_PAGE_COUNT){
return true;
}
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
try {
frame.setPage(rootNode);
@@ -263,8 +266,7 @@
public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException {
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false,
- largePageHelper);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
rootNode.acquireReadLatch();
try {
frame.setPage(rootNode);
@@ -394,13 +396,12 @@
//move the root page to the first data page if necessary
bufferCache.finishQueue();
if (!appendOnly) {
- ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true,
- largePageHelper);
+ ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
newRoot.acquireWriteLatch();
//root will be the highest frontier
NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1);
ICachedPage oldRoot = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false, largePageHelper);
+ BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false);
oldRoot.acquireReadLatch();
lastNodeFrontier.page = oldRoot;
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
index 87449eb..679ea2f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
@@ -64,14 +64,15 @@
}
private boolean positionToNextLeaf(boolean skipCurrent) throws HyracksDataException {
- while ((frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) && (currentPageId <= maxPageId)) {
- currentPageId++;
+ while (frame.getLevel() != 0 || skipCurrent || frame.getTupleCount() == 0) {
+ if (++currentPageId > maxPageId) {
+ break;
+ }
page.releaseReadLatch();
bufferCache.unpin(page);
- ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false,
- frame.getLargePageHelper());
+ ICachedPage nextPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
nextPage.acquireReadLatch();
page = nextPage;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index d9013d3..5668992 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -53,8 +53,7 @@
// scan entire file to determine pages in each level
int maxPageId = freePageManager.getMaxPage(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
try {
frame.setPage(page);
@@ -89,8 +88,7 @@
int pageId = remainingPageIds[index];
// pin & latch then immediately unlatch & unpin
- ICachedPage page = bufferCache.pin(BufferedFileHandle
- .getDiskPageId(fileId, pageId), false);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
page.releaseReadLatch();
bufferCache.unpin(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index 7a9e588..712acb8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -52,8 +52,7 @@
int maxPageId = freePageManager.getMaxPage(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false, leafFrame.getLargePageHelper());
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
try {
metaFrame.setPage(page);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index c4fce24..748dbfe 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -25,8 +25,8 @@
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.replication.IIOReplicationManager;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -69,11 +69,6 @@
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
return vbc.pin(dpid, newPage);
- }
-
- @Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- return vbc.pin(dpid, newPage, helper);
}
@Override
@@ -163,8 +158,9 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return vbc.confiscateLargePage(dpid, multiplier);
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ return vbc.confiscateLargePage(dpid, multiplier, extraBlockPageId);
}
@Override
@@ -218,7 +214,8 @@
}
@Override
- public void resizePage(ICachedPage page, int multiple) {
- vbc.resizePage(page, multiple);
+ public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ vbc.resizePage(page, multiplier, extraPageBlockHelper);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index e62f824..40c7734 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -34,8 +34,8 @@
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
import org.apache.hyracks.storage.common.buffercache.IQueueInfo;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -66,7 +66,7 @@
this.numPages = 2 * (numPages / 2) + 1;
buckets = new CacheBucket[this.numPages];
- pages = new ArrayList<VirtualPage>();
+ pages = new ArrayList<>();
nextFree = 0;
largePages = new AtomicInteger(0);
open = false;
@@ -168,11 +168,6 @@
}
@Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- return pin(dpid, newPage);
- }
-
- @Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
VirtualPage page = null;
int hash = hash(dpid);
@@ -213,6 +208,7 @@
synchronized (pages) {
if (nextFree >= pages.size()) {
page = new VirtualPage(allocator.allocate(pageSize, 1)[0]);
+ page.multiplier = 1;
pages.add(page);
} else {
page = pages.get(nextFree);
@@ -224,9 +220,9 @@
}
@Override
- public void resizePage(ICachedPage cPage, int multiplier) {
+ public void resizePage(ICachedPage cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper) {
ByteBuffer oldBuffer = cPage.getBuffer();
- int origMultiplier = oldBuffer.capacity() / pageSize;
+ int origMultiplier = cPage.getFrameSizeMultiplier();
if (origMultiplier == multiplier) {
// no-op
return;
@@ -253,6 +249,7 @@
largePages.getAndAdd(multiplier - origMultiplier);
}
((VirtualPage)cPage).buffer = newBuffer;
+ ((VirtualPage)cPage).multiplier = multiplier;
}
@Override
@@ -354,6 +351,7 @@
ByteBuffer buffer;
final ReadWriteLock latch;
volatile long dpid;
+ int multiplier;
VirtualPage next;
public VirtualPage(ByteBuffer buffer) {
@@ -406,6 +404,10 @@
throw new UnsupportedOperationException();
}
+ @Override
+ public int getFrameSizeMultiplier() {
+ return multiplier;
+ }
}
//These 4 methods aren't applicable here.
@@ -447,12 +449,13 @@
}
@Override
- public ICachedPage confiscatePage(long dpid) {
+ public ICachedPage confiscatePage(long dpid) throws HyracksDataException {
throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index 56827a5..84cb82e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -21,11 +21,9 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
@@ -34,6 +32,7 @@
import org.apache.hyracks.storage.am.rtree.api.IRTreePolicy;
import org.apache.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeFrame {
protected static final int pageNsnOff = flagOff + 1; // 22
@@ -50,7 +49,7 @@
public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders,
RTreePolicyType rtreePolicyType, boolean isPointMBR) {
- super(tupleWriter, new UnorderedSlotManager(), null);
+ super(tupleWriter, new UnorderedSlotManager());
this.mbrTuples = new ITreeIndexTupleReference[keyValueProviders.length];
for (int i = 0; i < keyValueProviders.length; i++) {
this.mbrTuples[i] = tupleWriter.createTupleReference();
@@ -116,7 +115,7 @@
@Override
public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
- IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
+ IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache)
throws HyracksDataException {
rtreePolicy.split(this, buf, rightFrame, slotManager, frameTuple, tuple, splitKey);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
index 02177f4..8798a70 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
@@ -61,8 +60,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
index 6373c7e..8573119 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
@@ -24,7 +24,6 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
@@ -61,8 +60,4 @@
return tupleWriterFactory;
}
- @Override
- public ILargePageHelper getLargePageHelper() {
- return null;
- }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index 84180aa..99edbc4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -155,8 +155,8 @@
private RTreeOpContext createOpContext(IModificationOperationCallback modificationCallback) {
return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
- (IRTreeInteriorFrame) interiorFrameFactory.createFrame(),
- freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback);
+ (IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager, cmpFactories,
+ modificationCallback);
}
private void insert(ITupleReference tuple, IIndexOperationContext ictx)
@@ -412,7 +412,7 @@
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
- ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+ ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, ctx,
bufferCache);
ctx.interiorFrame.setRightPage(rightPageId);
} else {
@@ -421,7 +421,7 @@
rightFrame.initBuffer((byte) 0);
rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
ctx.modificationCallback.found(null, tuple);
- ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx,
bufferCache);
ctx.leafFrame.setRightPage(rightPageId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index 107dc30..a04562f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -22,8 +22,10 @@
import java.util.ArrayList;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
@@ -32,8 +34,9 @@
import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
-public class RTreeOpContext implements IIndexOperationContext {
+public class RTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper {
private static final int INITIAL_TRAVERSE_LIST_SIZE = 100;
private static final int INITIAL_HEIGHT = 8;
public final MultiComparator cmp;
@@ -42,7 +45,8 @@
public IndexOperation op;
public ITreeIndexCursor cursor;
public RTreeCursorInitialState cursorInitialState;
- public ITreeIndexMetaDataFrame metaFrame;
+ public final IMetaDataPageManager freePageManager;
+ public final ITreeIndexMetaDataFrame metaFrame;
public RTreeSplitKey splitKey;
public ITupleReference tuple;
// Used to record the pageIds and pageLsns of the visited pages.
@@ -56,8 +60,8 @@
public IModificationOperationCallback modificationCallback;
public RTreeOpContext(IRTreeLeafFrame leafFrame, IRTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, IBinaryComparatorFactory[] cmpFactories,
- IModificationOperationCallback modificationCallback) {
+ IMetaDataPageManager freePageManager, IBinaryComparatorFactory[] cmpFactories,
+ IModificationOperationCallback modificationCallback) {
if (cmpFactories[0] != null) {
this.cmp = MultiComparator.create(cmpFactories);
@@ -67,7 +71,8 @@
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
- this.metaFrame = metaFrame;
+ this.freePageManager = freePageManager;
+ this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
this.modificationCallback = modificationCallback;
pathList = new PathList(INITIAL_HEIGHT, INITIAL_HEIGHT);
NSNUpdates = new ArrayList<ICachedPage>();
@@ -121,4 +126,14 @@
public void setModificationCallback(IModificationOperationCallback modificationCallback) {
this.modificationCallback = modificationCallback;
}
+
+ @Override
+ public int getFreeBlock(int size) throws HyracksDataException {
+ return freePageManager.getFreePageBlock(metaFrame, size);
+ }
+
+ @Override
+ public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException {
+ freePageManager.addFreePageBlock(metaFrame, blockPageId, size);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 201526b..ceb8ca2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -28,8 +28,10 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@@ -49,6 +51,7 @@
import org.apache.hyracks.storage.common.file.IFileMapManager;
public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
+
private static final Logger LOGGER = Logger.getLogger(BufferCache.class.getName());
private static final int MAP_FACTOR = 3;
@@ -68,6 +71,8 @@
private final Map<Integer, BufferedFileHandle> fileInfoMap;
private final Set<Integer> virtualFiles;
private final AsyncFIFOPageQueueManager fifoWriter;
+ private final Queue<BufferCacheHeaderHelper> headerPageCache = new ConcurrentLinkedQueue<>();
+
//DEBUG
private ArrayList<CachedPage> confiscatedPages;
private Lock confiscateLock;
@@ -124,6 +129,10 @@
return pageSize;
}
+ public int getPageSizeWithHeader() {
+ return pageSize + RESERVED_HEADER_BYTES;
+ }
+
@Override
public int getNumPages() {
return pageReplacementStrategy.getMaxAllowedNumPages();
@@ -176,11 +185,6 @@
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
- return pin(dpid, newPage, null);
- }
-
- @Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
// Calling the pinSanityCheck should be used only for debugging, since
// the synchronized block over the fileInfoMap is a hot spot.
if (DEBUG) {
@@ -193,9 +197,7 @@
try {
for (CachedPage c : confiscatedPages) {
if (c.dpid == dpid && c.confiscated.get()) {
- while(confiscatedPages.contains(c)){
- throw new IllegalStateException();
- }
+ throw new IllegalStateException();
}
}
} finally{
@@ -206,7 +208,7 @@
// disk.
synchronized (cPage) {
if (!cPage.valid) {
- read(cPage, helper);
+ read(cPage);
cPage.valid = true;
}
}
@@ -215,9 +217,8 @@
}
pageReplacementStrategy.notifyCachePageAccess(cPage);
if(DEBUG){
- pinnedPageOwner.put((CachedPage) cPage, Thread.currentThread().getStackTrace());
+ pinnedPageOwner.put(cPage, Thread.currentThread().getStackTrace());
}
- cPage.setLargePageHelper(helper);
return cPage;
}
@@ -565,26 +566,42 @@
return false;
}
- private void read(CachedPage cPage, ILargePageHelper helper) throws HyracksDataException {
+ private void read(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
cPage.buffer.clear();
- ioManager.syncRead(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
- cPage.buffer);
- if (helper != null) {
- int totalPages = helper.getSupplementalBlockNumPages(cPage) + 1;
+ BufferCacheHeaderHelper header = checkoutHeaderHelper();
+ try {
+ long bytesRead = ioManager.syncRead(fInfo.getFileHandle(),
+ getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)), header.prepareRead());
+
+ if (bytesRead != getPageSizeWithHeader()) {
+ if (bytesRead == -1) {
+ // disk order scan code seems to rely on this behavior, so silently return
+ return;
+ }
+ throw new HyracksDataException("Failed to read a complete page: " + bytesRead);
+ }
+ int totalPages = header.processRead(cPage);
+
if (totalPages > 1) {
- resizePage(cPage, totalPages);
+ pageReplacementStrategy.fixupCapacityOnLargeRead(cPage);
cPage.buffer.position(pageSize);
cPage.buffer.limit(totalPages * pageSize);
- ioManager.syncRead(fInfo.getFileHandle(), (long) helper.getSupplementalBlockPageId(cPage) * pageSize,
- cPage.buffer);
+ ioManager.syncRead(fInfo.getFileHandle(), getOffsetForPage(cPage.getExtraBlockPageId()), cPage.buffer);
}
+ } finally {
+ returnHeaderHelper(header);
}
}
+ private long getOffsetForPage(long pageId) {
+ return pageId * getPageSizeWithHeader();
+ }
+
@Override
- public void resizePage(ICachedPage cPage, int totalPages) {
- pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages);
+ public void resizePage(ICachedPage cPage, int totalPages, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages, extraPageBlockHelper);
}
BufferedFileHandle getFileInfo(CachedPage cPage) throws HyracksDataException {
@@ -600,6 +617,17 @@
return fInfo;
}
}
+ private BufferCacheHeaderHelper checkoutHeaderHelper() {
+ BufferCacheHeaderHelper helper = headerPageCache.poll();
+ if (helper == null) {
+ helper = new BufferCacheHeaderHelper(pageSize);
+ }
+ return helper;
+ }
+
+ private void returnHeaderHelper(BufferCacheHeaderHelper buffer) {
+ headerPageCache.offer(buffer);
+ }
void write(CachedPage cPage) throws HyracksDataException {
BufferedFileHandle fInfo = getFileInfo(cPage);
@@ -607,21 +635,29 @@
synchronized (fInfo) {
if (!fInfo.fileHasBeenDeleted()) {
ByteBuffer buf = cPage.buffer.duplicate();
- buf.position(0);
- buf.limit(pageSize);
- ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
- buf);
- if (cPage.largePageHelper != null) {
- int totalPages = cPage.largePageHelper.getSupplementalBlockNumPages(cPage) + 1;
- if (totalPages > 1) {
- buf.limit(totalPages * pageSize);
- long offset = (long) cPage.largePageHelper.getSupplementalBlockPageId(cPage) * pageSize;
- ioManager.syncWrite(fInfo.getFileHandle(), offset, buf);
+ final int totalPages = cPage.getFrameSizeMultiplier();
+ final int extraBlockPageId = cPage.getExtraBlockPageId();
+ final boolean contiguousLargePages = (BufferedFileHandle.getPageId(cPage.dpid) + 1) == extraBlockPageId;
+ BufferCacheHeaderHelper header = checkoutHeaderHelper();
+ try {
+ buf.limit(contiguousLargePages ? pageSize * totalPages : pageSize);
+ buf.position(0);
+ long bytesWritten = ioManager.syncWrite(fInfo.getFileHandle(),
+ getOffsetForPage(BufferedFileHandle.getPageId(cPage.dpid)),
+ header.prepareWrite(cPage, buf));
+
+ if (bytesWritten !=
+ (contiguousLargePages ? pageSize * (totalPages - 1) : 0) + getPageSizeWithHeader()) {
+ throw new HyracksDataException("Failed to write completely: " + bytesWritten);
}
- assert buf.capacity() == (pageSize * totalPages);
- } else {
- assert buf.capacity() == pageSize;
+ } finally {
+ returnHeaderHelper(header);
}
+ if (totalPages > 1 && !contiguousLargePages) {
+ buf.limit(totalPages * pageSize);
+ ioManager.syncWrite(fInfo.getFileHandle(), getOffsetForPage(extraBlockPageId), buf);
+ }
+ assert buf.capacity() == (pageSize * totalPages);
}
}
}
@@ -1134,9 +1170,9 @@
throw new HyracksDataException("No such file mapped for fileId:" + fileId);
}
if(DEBUG) {
- assert ioManager.getSize(fInfo.getFileHandle()) % getPageSize() == 0;
+ assert ioManager.getSize(fInfo.getFileHandle()) % getPageSizeWithHeader() == 0;
}
- return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSize());
+ return (int) (ioManager.getSize(fInfo.getFileHandle()) / getPageSizeWithHeader());
}
}
@@ -1151,8 +1187,11 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier));
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ ICachedPage cachedPage = confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier));
+ ((ICachedPageInternal)cachedPage).setExtraBlockPageId(extraBlockPageId);
+ return cachedPage;
}
private ICachedPage confiscatePage(long dpid, Supplier<ICachedPageInternal> victimSupplier)
@@ -1236,7 +1275,6 @@
// if we found a page after all that, go ahead and finish
if (returnPage != null) {
((CachedPage) returnPage).confiscated.set(true);
- ((CachedPage) returnPage).setLargePageHelper(null);
if (DEBUG) {
confiscateLock.lock();
try{
@@ -1336,7 +1374,6 @@
}
}
pageReplacementStrategy.adviseWontNeed(cPage);
- cPage.largePageHelper = null;
}
@Override
@@ -1390,4 +1427,41 @@
}
}
+ static class BufferCacheHeaderHelper {
+ private static final int FRAME_MULTIPLIER_OFF = 0;
+ private static final int EXTRA_BLOCK_PAGE_ID_OFF = FRAME_MULTIPLIER_OFF + 4; // 4
+
+ private final ByteBuffer buf;
+ private final ByteBuffer [] array;
+
+ private BufferCacheHeaderHelper(int pageSize) {
+ buf = ByteBuffer.allocate(RESERVED_HEADER_BYTES + pageSize);
+ array = new ByteBuffer[] { buf, null };
+ }
+
+ private ByteBuffer[] prepareWrite(CachedPage cPage, ByteBuffer pageBuffer) {
+ buf.position(0);
+ buf.limit(RESERVED_HEADER_BYTES);
+ buf.putInt(FRAME_MULTIPLIER_OFF, cPage.getFrameSizeMultiplier());
+ buf.putInt(EXTRA_BLOCK_PAGE_ID_OFF, cPage.getExtraBlockPageId());
+ array[1] = pageBuffer;
+ return array;
+ }
+
+ private ByteBuffer prepareRead() {
+ buf.position(0);
+ buf.limit(buf.capacity());
+ return buf;
+ }
+
+ private int processRead(CachedPage cPage) {
+ buf.position(RESERVED_HEADER_BYTES);
+ cPage.buffer.position(0);
+ cPage.buffer.put(buf);
+ int multiplier = buf.getInt(FRAME_MULTIPLIER_OFF);
+ cPage.setFrameSizeMultiplier(multiplier);
+ cPage.setExtraBlockPageId(buf.getInt(EXTRA_BLOCK_PAGE_ID_OFF));
+ return multiplier;
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
index ddab742..4541d52 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
@@ -39,7 +39,11 @@
volatile boolean valid;
final AtomicBoolean confiscated;
private IQueueInfo queueInfo;
- ILargePageHelper largePageHelper;
+ private int multiplier;
+ private int extraBlockPageId;
+ // DEBUG
+ private static final boolean DEBUG = false;
+ private final StackTraceElement[] ctorStack;
//Constructor for making dummy entry for FIFO queue
public CachedPage(){
@@ -52,6 +56,7 @@
queueInfo = null;
replacementStrategyObject = null;
latch =null;
+ ctorStack = DEBUG ? new Throwable().getStackTrace() : null;
}
public CachedPage(int cpid, ByteBuffer buffer, IPageReplacementStrategy pageReplacementStrategy) {
@@ -66,6 +71,7 @@
valid = false;
confiscated = new AtomicBoolean(false);
queueInfo = null;
+ ctorStack = DEBUG ? new Throwable().getStackTrace() : null;
}
public void reset(long dpid) {
@@ -75,7 +81,6 @@
confiscated.set(false);
pageReplacementStrategy.notifyCachePageReset(this);
queueInfo = null;
- largePageHelper = null;
}
public void invalidate() {
@@ -154,6 +159,26 @@
return dpid;
}
+ @Override
+ public int getFrameSizeMultiplier() {
+ return multiplier;
+ }
+
+ @Override
+ public void setFrameSizeMultiplier(int multiplier) {
+ this.multiplier = multiplier;
+ }
+
+ @Override
+ public void setExtraBlockPageId(int extraBlockPageId) {
+ this.extraBlockPageId = extraBlockPageId;
+ }
+
+ @Override
+ public int getExtraBlockPageId() {
+ return extraBlockPageId;
+ }
+
CachedPage getNext() {
return next;
}
@@ -161,13 +186,4 @@
void setNext(CachedPage next) {
this.next = next;
}
-
- public ILargePageHelper getLargePageHelper() {
- return largePageHelper;
- }
-
- public void setLargePageHelper(ILargePageHelper largePageHelper) {
- this.largePageHelper = largePageHelper;
- }
-
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
index ea27757..e397d66 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
@@ -25,6 +25,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
private static final Logger LOGGER = Logger.getLogger(ClockPageReplacementStrategy.class.getName());
private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3;
@@ -80,13 +82,13 @@
@Override
public ICachedPageInternal findVictim(int multiplier) {
- while (numPages.get() + (multiplier - 1) >= maxAllowedNumPages) {
+ while (numPages.get() + multiplier > maxAllowedNumPages) {
// TODO: is dropping pages on the floor enough to adhere to memory budget?
ICachedPageInternal victim = findVictimByEviction();
if (victim == null) {
return null;
}
- int multiple = victim.getBuffer().capacity() / pageSize;
+ int multiple = victim.getFrameSizeMultiplier();
if (multiple == multiplier) {
return victim;
} else if (bufferCache.removePage(victim)) {
@@ -153,50 +155,79 @@
cpId = cpIdCounter.getAndIncrement();
}
CachedPage cPage = new CachedPage(cpId, allocator.allocate(pageSize * multiplier, 1)[0], this);
+ cPage.setFrameSizeMultiplier(multiplier);
bufferCache.addPage(cPage);
numPages.getAndAdd(multiplier);
return cPage;
}
@Override
- public void resizePage(ICachedPageInternal cPage, int multiplier) {
- ByteBuffer oldBuffer = cPage.getBuffer();
- int origMultiplier = oldBuffer.capacity() / pageSize;
+ public void resizePage(ICachedPageInternal cPage, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ int origMultiplier = cPage.getFrameSizeMultiplier();
if (origMultiplier == multiplier) {
// no-op
return;
}
final int newSize = pageSize * multiplier;
+ ByteBuffer oldBuffer = ((CachedPage)cPage).buffer;
oldBuffer.position(0);
+ final int delta = multiplier - origMultiplier;
if (multiplier < origMultiplier) {
oldBuffer.limit(newSize);
+ final int gap = -delta;
+ // we return the unused portion of our block to the page manager
+ extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId() + gap, gap);
} else {
- while (numPages.get() + (multiplier - origMultiplier) >= maxAllowedNumPages) {
- ICachedPageInternal victim = findVictimByEviction();
- if (victim != null) {
- int multiple = victim.getBuffer().capacity() / pageSize;
- if (bufferCache.removePage(victim)) {
- // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
- cpIdFreeList.add(victim.getCachedPageId());
- numPages.getAndAdd(-multiple);
- }
- } else {
- // TODO(mblow): what should we do with we need to resize and don't have the budget?
- // we don't have the budget to resize- proceed anyway, but log
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by "
- + (numPages.get() + (multiplier - origMultiplier) - maxAllowedNumPages)
- + " pages in order to satisfy large page read");
- }
- break;
-
- }
+ ensureBudgetForLargePages(delta);
+ if (origMultiplier != 1) {
+ // return the old block to the page manager
+ extraPageBlockHelper.returnFreePageBlock(cPage.getExtraBlockPageId(), origMultiplier);
}
+ cPage.setExtraBlockPageId(extraPageBlockHelper.getFreeBlock(multiplier));
}
+ cPage.setFrameSizeMultiplier(multiplier);
ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0];
newBuffer.put(oldBuffer);
- numPages.getAndAdd(multiplier - origMultiplier);
+ numPages.getAndAdd(delta);
((CachedPage) cPage).buffer = newBuffer;
+ }
+
+ @Override
+ public void fixupCapacityOnLargeRead(ICachedPageInternal cPage)
+ throws HyracksDataException {
+ ByteBuffer oldBuffer = ((CachedPage) cPage).buffer;
+ final int multiplier = cPage.getFrameSizeMultiplier();
+ final int newSize = pageSize * multiplier;
+ final int delta = multiplier - 1;
+ oldBuffer.position(0);
+ ensureBudgetForLargePages(delta);
+ ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0];
+ newBuffer.put(oldBuffer);
+ numPages.getAndAdd(delta);
+ ((CachedPage) cPage).buffer = newBuffer;
+ }
+
+ private void ensureBudgetForLargePages(int delta) {
+ while (numPages.get() + delta >= maxAllowedNumPages) {
+ ICachedPageInternal victim = findVictimByEviction();
+ if (victim != null) {
+ final int victimMultiplier = victim.getFrameSizeMultiplier();
+ if (bufferCache.removePage(victim)) {
+ cpIdFreeList.add(victim.getCachedPageId());
+ numPages.getAndAdd(-victimMultiplier);
+ }
+ } else {
+ // we don't have the budget to resize- proceed anyway, but log
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by "
+ + (numPages.get() + delta - maxAllowedNumPages)
+ + " pages in order to satisfy large page read");
+ }
+ break;
+
+ }
+ }
}
//derived from RoundRobinAllocationPolicy in Apache directmemory
@@ -232,4 +263,5 @@
//make the page appear as if it wasn't accessed even if it was
getPerPageObject(cPage).set(false);
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index c78012b..4a41ec0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -88,13 +88,6 @@
}
@Override
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(dpid, newPage, helper);
- pinCount.addAndGet(1);
- return page;
- }
-
- @Override
public void unpin(ICachedPage page) throws HyracksDataException {
bufferCache.unpin(page);
unpinCount.addAndGet(1);
@@ -204,8 +197,9 @@
}
@Override
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
- return bufferCache.confiscateLargePage(dpid, multiplier);
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId)
+ throws HyracksDataException {
+ return bufferCache.confiscateLargePage(dpid, multiplier, extraBlockPageId);
}
@Override
@@ -262,7 +256,8 @@
}
@Override
- public void resizePage(ICachedPage page, int multiple) {
- bufferCache.resizePage(page, multiple);
+ public void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException {
+ bufferCache.resizePage(page, multiplier, extraPageBlockHelper);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index feac69e..ca87673 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -24,8 +24,9 @@
public interface IBufferCache {
- public static final long INVALID_DPID = -1l;
- public static final int INVALID_PAGEID = -1;
+ long INVALID_DPID = -1l;
+ int INVALID_PAGEID = -1;
+ int RESERVED_HEADER_BYTES = 8;
public void createFile(FileReference fileRef) throws HyracksDataException;
@@ -43,8 +44,6 @@
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
- public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException;
-
public void unpin(ICachedPage page) throws HyracksDataException;
public void flushDirtyPage(ICachedPage page) throws HyracksDataException;
@@ -53,7 +52,7 @@
public ICachedPage confiscatePage(long dpid) throws HyracksDataException;
- public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException;
+ public ICachedPage confiscateLargePage(long dpid, int multiplier, int extraBlockPageId) throws HyracksDataException;
public void returnPage(ICachedPage page);
@@ -61,7 +60,7 @@
public void force(int fileId, boolean metadata) throws HyracksDataException;
- public int getPageSize();
+ int getPageSize();
public int getNumPages();
@@ -85,5 +84,6 @@
void purgeHandle(int fileId) throws HyracksDataException;
- void resizePage(ICachedPage page, int multiple);
+ void resizePage(ICachedPage page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
index 12dde49..8107234 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
@@ -21,19 +21,22 @@
import java.nio.ByteBuffer;
public interface ICachedPage {
- public ByteBuffer getBuffer();
- public void acquireReadLatch();
+ ByteBuffer getBuffer();
- public void releaseReadLatch();
+ void acquireReadLatch();
- public void acquireWriteLatch();
+ void releaseReadLatch();
- public void releaseWriteLatch(boolean markDirty);
+ void acquireWriteLatch();
- public boolean confiscated();
+ void releaseWriteLatch(boolean markDirty);
- public IQueueInfo getQueueInfo();
+ boolean confiscated();
- public void setQueueInfo(IQueueInfo queueInfo);
+ IQueueInfo getQueueInfo();
+
+ void setQueueInfo(IQueueInfo queueInfo);
+
+ int getFrameSizeMultiplier();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
index 6a6c2f5..c500286 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
@@ -27,4 +27,9 @@
public boolean isGoodVictim();
+ void setFrameSizeMultiplier(int multiplier);
+
+ int getExtraBlockPageId();
+
+ void setExtraBlockPageId(int extraBlockPageId);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
similarity index 77%
copy from hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
copy to hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
index 22bec97..ad7f2f6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
@@ -18,8 +18,9 @@
*/
package org.apache.hyracks.storage.common.buffercache;
-public interface ILargePageHelper {
+import org.apache.hyracks.api.exceptions.HyracksDataException;
- int getSupplementalBlockNumPages(CachedPage cPage);
- int getSupplementalBlockPageId(CachedPage cPage);
+public interface IExtraPageBlockHelper {
+ int getFreeBlock(int size) throws HyracksDataException;
+ void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
index 1a59a65..d3bcce5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.storage.common.buffercache;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public interface IPageReplacementStrategy {
public Object createPerPageStrategyObject(int cpid);
@@ -37,9 +39,13 @@
public int getNumPages();
+ void fixupCapacityOnLargeRead(ICachedPageInternal cPage)
+ throws HyracksDataException;
+
public int getPageSize();
public int getMaxAllowedNumPages();
- public void resizePage(ICachedPageInternal page, int multiplier);
+ void resizePage(ICachedPageInternal page, int multiplier, IExtraPageBlockHelper extraPageBlockHelper)
+ throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index 840acd5..4420fb0 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -23,7 +23,6 @@
import java.util.Random;
import java.util.logging.Level;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.junit.Assert;
import org.junit.Test;
@@ -136,8 +135,7 @@
int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
bufferCache.openFile(btreeFileId);
IHyracksTaskContext ctx = harness.getHyracksTaskContext();
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true);
try {
ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
index cce2f82..01863ca 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
@@ -24,7 +24,6 @@
import java.util.Random;
import java.util.logging.Level;
-import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
import org.junit.Test;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -90,8 +89,7 @@
}
try {
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false,
- BTreeLargeFrameHelper.INSTANCE);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
LatchType latch = null;
switch (fta) {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
index 2330e4d..e85ff9e 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
@@ -18,15 +18,9 @@
*/
package org.apache.hyracks.storage.common;
-import static org.junit.Assert.fail;
-
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.io.FileReference;
@@ -34,12 +28,18 @@
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IIOManager.FileReadWriteMode;
import org.apache.hyracks.api.io.IIOManager.FileSyncMode;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
import org.apache.hyracks.test.support.TestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
public class BufferCacheRegressionTest {
protected static final String tmpDir = System.getProperty("java.io.tmpdir");
@@ -126,9 +126,9 @@
FileReference testFileRef = new FileReference(new File(fileName));
IFileHandle testFileHandle = ioManager.open(testFileRef, FileReadWriteMode.READ_ONLY,
FileSyncMode.METADATA_SYNC_DATA_SYNC);
- ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE);
+ ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE + BufferCache.RESERVED_HEADER_BYTES);
ioManager.syncRead(testFileHandle, 0, testBuffer);
- for (int i = 0; i < testBuffer.capacity(); i++) {
+ for (int i = BufferCache.RESERVED_HEADER_BYTES; i < testBuffer.capacity(); i++) {
if (deleteFile) {
// We deleted the file. We expect to see a clean buffer.
if (testBuffer.get(i) == Byte.MAX_VALUE) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 4:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1647/
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1640/
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 2:
(6 comments)
Very minor comments
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
File hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java:
Line 46: public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
looking at this method definition, no one would guess that the dataArray is always a two parameters (header and data).
I think we can do better :) (at least, a comment explaining)
Line 50: public long syncRead(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException;
Remove unused? or comment future use?
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
File hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java:
Line 189: @Override
remove method?
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
File hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java:
Line 47:
Am I seeing tabs??
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
File hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java:
Line 46: ITreeIndexFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame)
Huh! how come Sonar doesn't detect those?
https://asterix-gerrit.ics.uci.edu/#/c/920/2/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
File hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java:
Line 228: // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
I believe it is enough. would be nice to find a better source to confirm
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/1641/
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 2:
WARNING: THIS CHANGE CONTAINS CROSS-PRODUCT CHANGES IN:
* asterixdb
* hyracks-fullstack
PLEASE REVIEW CAREFULLY AND LOOK FOR API CHANGES!
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "abdullah alamoudi (Code Review)" <do...@asterixdb.incubator.apache.org>.
abdullah alamoudi has posted comments on this change.
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
Patch Set 4: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Michael Blow has uploaded a new patch set (#2).
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj
Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart
- large page information managed by buffer cache, outside of page
- size header written out a prefix to pages
Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
54 files changed, 531 insertions(+), 419 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/920/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: ASTERIXDB-1474: Cannot query dataset with big obj
Posted by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/920
to look at the new patch set (#4).
Change subject: ASTERIXDB-1474: Cannot query dataset with big obj
......................................................................
ASTERIXDB-1474: Cannot query dataset with big obj
Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart
- large page information managed by buffer cache, outside of page
- size header written out a prefix to pages
Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql
C asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql
R asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql
A asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh
A asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
D hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/TreeIndexDiskOrderScanCursor.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeOpContext.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
C hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IExtraPageBlockHelper.java
M hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
M hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheRegressionTest.java
54 files changed, 496 insertions(+), 447 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/20/920/4
--
To view, visit https://asterix-gerrit.ics.uci.edu/920
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I2f47c551a053e193519a4337c338934f3e171afd
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mi...@couchbase.com>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>