You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/12/16 07:49:47 UTC
[5/9] asterixdb git commit: Remove Append Only Flag
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 5ac203e..f8539cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -27,7 +27,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
@@ -51,7 +51,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
protected final IBufferCache bufferCache;
protected final IFileMapProvider fileMapProvider;
- protected final IMetaDataPageManager freePageManager;
+ protected final IPageManager freePageManager;
protected final ITreeIndexFrameFactory interiorFrameFactory;
protected final ITreeIndexFrameFactory leafFrameFactory;
@@ -67,12 +67,11 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
//was never activated or failed to activate in try/finally blocks, as there's no way to know if
//an index is activated or not from the outside.
protected boolean hasEverBeenActivated = false;
- protected boolean appendOnly = false;
protected int bulkloadLeafStart = 0;
public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IMetaDataPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+ IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
FileReference file) {
this.bufferCache = bufferCache;
@@ -85,18 +84,13 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
this.file = file;
}
+ @Override
public synchronized void create() throws HyracksDataException {
- create(false);
- }
-
- private synchronized void create(boolean appendOnly) throws HyracksDataException {
if (isActive) {
throw new HyracksDataException("Failed to create the index since it is activated.");
}
-
- boolean fileIsMapped = false;
synchronized (fileMapProvider) {
- fileIsMapped = fileMapProvider.isMapped(file);
+ boolean fileIsMapped = fileMapProvider.isMapped(file);
if (!fileIsMapped) {
bufferCache.createFile(file);
}
@@ -114,55 +108,22 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
}
freePageManager.open(fileId);
- if (!appendOnly) {
- initEmptyTree();
- freePageManager.close();
- } else {
- this.appendOnly = true;
- initCachedMetadataPage();
- }
- setRootPage(appendOnly);
+ freePageManager.init(interiorFrameFactory, leafFrameFactory);
+ setRootPage();
+ freePageManager.close();
bufferCache.closeFile(fileId);
}
- private void initEmptyTree() throws HyracksDataException {
- ITreeIndexFrame frame = leafFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
- freePageManager.init(metaFrame, rootPage);
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- rootNode.acquireWriteLatch();
- try {
- frame.setPage(rootNode);
- frame.initBuffer((byte) 0);
- } finally {
- rootNode.releaseWriteLatch(true);
- bufferCache.unpin(rootNode);
- }
- }
-
- private void setRootPage(boolean appendOnly) throws HyracksDataException {
- if (!appendOnly) {
- // regular or empty tree
- rootPage = 1;
- bulkloadLeafStart = 2;
- } else {
- //root page is stored in MD page
- rootPage = freePageManager.getRootPage();
- //leaves start from the very beginning of the file.
- bulkloadLeafStart = 0;
- }
- }
-
- private void initCachedMetadataPage() throws HyracksDataException {
- ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
- freePageManager.init(metaFrame);
+ private void setRootPage() throws HyracksDataException {
+ rootPage = freePageManager.getRootPageId();
+ bulkloadLeafStart = freePageManager.getBulkLoadLeaf();
}
+ @Override
public synchronized void activate() throws HyracksDataException {
if (isActive) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
-
boolean fileIsMapped = false;
synchronized (fileMapProvider) {
fileIsMapped = fileMapProvider.isMapped(file);
@@ -182,23 +143,14 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
}
}
freePageManager.open(fileId);
- int mdPageLoc = freePageManager.getFirstMetadataPage();
- ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
- int numPages = freePageManager.getMaxPage(metaFrame);
- if (mdPageLoc > 1 || (mdPageLoc == 1 && numPages <= MINIMAL_TREE_PAGE_COUNT - 1)) { //md page doesn't count
- appendOnly = true;
- } else {
- appendOnly = false;
- }
- setRootPage(appendOnly);
-
+ setRootPage();
// TODO: Should probably have some way to check that the tree is physically consistent
// or that the file we just opened actually is a tree
-
isActive = true;
hasEverBeenActivated = true;
}
+ @Override
public synchronized void deactivate() throws HyracksDataException {
if (!isActive && hasEverBeenActivated) {
throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
@@ -217,6 +169,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
}
+ @Override
public synchronized void destroy() throws HyracksDataException {
if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
@@ -230,33 +183,20 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
fileId = -1;
}
+ @Override
public synchronized void clear() throws HyracksDataException {
if (!isActive) {
throw new HyracksDataException("Failed to clear the index since it is not activated.");
}
- initEmptyTree();
+ freePageManager.init(interiorFrameFactory, leafFrameFactory);
+ setRootPage();
}
public boolean isEmptyTree(ITreeIndexFrame frame) throws HyracksDataException {
if (rootPage == -1) {
return true;
}
- if (freePageManager.appendOnlyMode() && bufferCache.getNumPagesOfFile(fileId) <= MINIMAL_TREE_PAGE_COUNT) {
- return true;
- }
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
- rootNode.acquireReadLatch();
- try {
- frame.setPage(rootNode);
- if (frame.getLevel() == 0 && frame.getTupleCount() == 0) {
- return true;
- } else {
- return false;
- }
- } finally {
- rootNode.releaseReadLatch();
- bufferCache.unpin(rootNode);
- }
+ return freePageManager.isEmpty(frame, rootPage);
}
public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException {
@@ -271,6 +211,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
}
}
+ @Override
public int getFileId() {
return fileId;
}
@@ -279,30 +220,37 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
return file;
}
+ @Override
public IBufferCache getBufferCache() {
return bufferCache;
}
+ @Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
return interiorFrameFactory;
}
+ @Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
return leafFrameFactory;
}
+ @Override
public IBinaryComparatorFactory[] getComparatorFactories() {
return cmpFactories;
}
- public IMetaDataPageManager getMetaManager() {
+ @Override
+ public IPageManager getPageManager() {
return freePageManager;
}
+ @Override
public int getRootPageId() {
return rootPage;
}
+ @Override
public int getFieldCount() {
return fieldCount;
}
@@ -312,7 +260,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
protected final int slotSize;
protected final int leafMaxBytes;
protected final int interiorMaxBytes;
- protected final ArrayList<NodeFrontier> nodeFrontiers = new ArrayList<NodeFrontier>();
+ protected final ArrayList<NodeFrontier> nodeFrontiers = new ArrayList<>();
protected final ITreeIndexMetaDataFrame metaFrame;
protected final ITreeIndexTupleWriter tupleWriter;
protected ITreeIndexFrame leafFrame;
@@ -321,22 +269,14 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
// HDFS. Since loading this tree relies on the root page actually being at that point, no further inserts into
// that tree are allowed. Currently, this is not enforced.
protected boolean releasedLatches;
- public boolean appendOnly = false;
protected final IFIFOPageQueue queue;
protected List<ICachedPage> pagesToWrite;
- public AbstractTreeIndexBulkLoader(float fillFactor, boolean appendOnly)
+ public AbstractTreeIndexBulkLoader(float fillFactor)
throws TreeIndexException, HyracksDataException {
- //Initialize the tree
- if (appendOnly) {
- create(appendOnly);
- this.appendOnly = appendOnly;
- activate();
- }
-
leafFrame = leafFrameFactory.createFrame();
interiorFrame = interiorFrameFactory.createFrame();
- metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+ metaFrame = freePageManager.createMetadataFrame();
queue = bufferCache.createFIFOQueue();
@@ -352,23 +292,24 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
tupleWriter = leafFrame.getTupleWriter();
NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference());
- leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
+ leafFrontier.pageId = freePageManager.takePage(metaFrame);
leafFrontier.page = bufferCache
.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId));
interiorFrame.setPage(leafFrontier.page);
interiorFrame.initBuffer((byte) 0);
- interiorMaxBytes = (int) ((float) interiorFrame.getBuffer().capacity() * fillFactor);
+ interiorMaxBytes = (int) (interiorFrame.getBuffer().capacity() * fillFactor);
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
- leafMaxBytes = (int) ((float) leafFrame.getBuffer().capacity() * fillFactor);
+ leafMaxBytes = (int) (leafFrame.getBuffer().capacity() * fillFactor);
slotSize = leafFrame.getSlotSize();
nodeFrontiers.add(leafFrontier);
pagesToWrite = new ArrayList<>();
}
+ @Override
public abstract void add(ITupleReference tuple) throws IndexException, HyracksDataException;
protected void handleException() throws HyracksDataException {
@@ -387,33 +328,8 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
@Override
public void end() throws HyracksDataException {
- //move the root page to the first data page if necessary
bufferCache.finishQueue();
- if (!appendOnly) {
- ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- newRoot.acquireWriteLatch();
- //root will be the highest frontier
- NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1);
- ICachedPage oldRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId),
- false);
- oldRoot.acquireReadLatch();
- lastNodeFrontier.page = oldRoot;
- try {
- System.arraycopy(lastNodeFrontier.page.getBuffer().array(), 0, newRoot.getBuffer().array(), 0,
- lastNodeFrontier.page.getBuffer().capacity());
- } finally {
- newRoot.releaseWriteLatch(true);
- bufferCache.flushDirtyPage(newRoot);
- bufferCache.unpin(newRoot);
- oldRoot.releaseReadLatch();
- bufferCache.unpin(oldRoot);
-
- // register old root as a free page
- freePageManager.addFreePage(metaFrame, lastNodeFrontier.pageId);
-
- }
- }
- freePageManager.setRootPage(rootPage);
+ freePageManager.setRootPageId(rootPage);
}
protected void addLevel() throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index b4239d7..3a346c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -23,7 +23,7 @@ import java.util.Random;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.util.MathUtil;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.common.arraylist.IntArrayList;
@@ -33,13 +33,13 @@ import org.apache.hyracks.storage.common.file.BufferedFileHandle;
public class TreeIndexBufferCacheWarmup {
private final IBufferCache bufferCache;
- private final IMetaDataPageManager freePageManager;
+ private final IMetadataPageManager freePageManager;
private final int fileId;
private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<>();
private final Random rnd = new Random();
public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
- IMetaDataPageManager freePageManager, int fileId) {
+ IMetadataPageManager freePageManager, int fileId) {
this.bufferCache = bufferCache;
this.freePageManager = freePageManager;
this.fileId = fileId;
@@ -51,7 +51,7 @@ public class TreeIndexBufferCacheWarmup {
bufferCache.openFile(fileId);
// scan entire file to determine pages in each level
- int maxPageId = freePageManager.getMaxPage(metaFrame);
+ int maxPageId = freePageManager.getMaxPageId(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
index 8a28f41..cd4854f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
@@ -20,7 +20,6 @@ package org.apache.hyracks.storage.am.common.util;
import java.text.DecimalFormat;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
@@ -56,11 +55,10 @@ public class TreeIndexStats {
}
}
- public void add(ITreeIndexMetaDataFrame metaFrame,
- IMetaDataPageManager freePageManager) {
- if (freePageManager.isFreePage(metaFrame)) {
+ public void add(ITreeIndexMetaDataFrame metaFrame) {
+ if (metaFrame.isFreePage()) {
freePages++;
- } else if (freePageManager.isMetaPage(metaFrame)) {
+ } else if (metaFrame.isMetadataPage()) {
metaPages++;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index c24a90b..0dfac84 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -19,7 +19,7 @@
package org.apache.hyracks.storage.am.common.util;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -30,12 +30,12 @@ public class TreeIndexStatsGatherer {
private final TreeIndexStats treeIndexStats = new TreeIndexStats();
private final IBufferCache bufferCache;
- private final IMetaDataPageManager freePageManager;
+ private final IPageManager freePageManager;
private final int fileId;
private final int rootPage;
public TreeIndexStatsGatherer(IBufferCache bufferCache,
- IMetaDataPageManager freePageManager, int fileId, int rootPage) {
+ IPageManager freePageManager, int fileId, int rootPage) {
this.bufferCache = bufferCache;
this.freePageManager = freePageManager;
this.fileId = fileId;
@@ -50,7 +50,7 @@ public class TreeIndexStatsGatherer {
treeIndexStats.begin();
- int maxPageId = freePageManager.getMaxPage(metaFrame);
+ int maxPageId = freePageManager.getMaxPageId(metaFrame);
for (int pageId = 0; pageId <= maxPageId; pageId++) {
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
page.acquireReadLatch();
@@ -72,7 +72,7 @@ public class TreeIndexStatsGatherer {
treeIndexStats.add(interiorFrame);
}
} else {
- treeIndexStats.add(metaFrame, freePageManager);
+ treeIndexStats.add(metaFrame);
}
} finally {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index 9d9ed23..b11b364 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -22,6 +22,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -70,7 +71,8 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), getVersion(), durable);
+ ioOpCallbackFactory.createIOOperationCallback(), getVersion(), durable,
+ (IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
public int getVersion() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index 95fdbf1..dfbeb11 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -22,6 +22,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
@@ -82,7 +83,8 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version, durable);
+ ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version, durable,
+ (IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
public int getTargetVersion() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 25468a3..df029c7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -26,6 +26,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -77,6 +78,6 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields, durable);
+ btreeFields, filterFields, durable, (IMetadataPageManagerFactory) opDesc.getPageManagerFactory());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index ca269de..c79205e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -36,7 +36,7 @@ import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.am.common.api.IIndexCursor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
@@ -519,7 +519,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
frameTupleWriterFactory = ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory()
.getTupleWriterFactory();
bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput, numElementsHint, false, true);
+ verifyInput, numElementsHint, false);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
@@ -655,7 +655,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
@Override
- public IMetaDataPageManager getMetaManager() {
+ public IMetadataPageManager getPageManager() {
return null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 6181299..8e29ef2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -39,7 +39,7 @@ import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.am.common.api.IIndexCursor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
@@ -582,7 +582,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- public IMetaDataPageManager getMetaManager() {
+ public IMetadataPageManager getPageManager() {
// This method should never be called for disk only indexes
return null;
}
@@ -616,11 +616,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory.createLSMComponentInstance(
new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
if (createComponent) {
+ component.getBTree().create();
+ component.getBuddyBTree().create();
component.getBloomFilter().create();
- } else {
- component.getBTree().activate();
- component.getBuddyBTree().activate();
}
+ component.getBTree().activate();
+ component.getBuddyBTree().activate();
component.getBloomFilter().activate();
return component;
}
@@ -685,9 +686,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// Create the three loaders
btreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBTree()
- .createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true);
+ .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
buddyBtreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree()
- .createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true);
+ .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
bloomFilterFalsePositiveRate);
@@ -887,16 +888,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
- if (!appendOnly) {
- throw new IndexException("LSM Indices do not support in-place inserts");
- } else {
- return createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
- }
- }
-
- @Override
public void allocateMemoryComponents() throws HyracksDataException {
//do nothing since external index never use memory components
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 9d12562..66540ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -41,8 +41,8 @@ import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
import org.apache.hyracks.storage.am.common.api.IIndexCursor;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
@@ -71,7 +71,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualMetaDataPageManager;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -121,7 +121,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(
new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
- new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), interiorFrameFactory,
+ new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory,
insertLeafFrameFactory, cmpFactories, fieldCount,
ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)),
virtualBufferCache, i == 0 ? true : false,
@@ -130,8 +130,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
++i;
}
componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterFactory);
- bulkLoadComponentFactory =
- new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, filterFactory);
+ bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory,
+ filterFactory);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
this.hasBloomFilter = needKeyDupCheck;
@@ -187,9 +187,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
LSMBTreeDiskComponent component;
try {
- component =
- createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
+ component = createDiskComponent(componentFactory, lsmComonentFileReference
+ .getInsertIndexFileReference(),
+ lsmComonentFileReference.getBloomFilterFileReference(), false);
} catch (IndexException e) {
throw new HyracksDataException(e);
}
@@ -461,7 +461,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getBTreeFlushTarget(),
flushOp.getBloomFilterFlushTarget(), true);
- IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false, numElements, false, true);
+ IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false, numElements, false);
IIndexBulkLoader builder = null;
if (hasBloomFilter) {
builder = component.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(),
@@ -514,8 +514,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
- LSMComponentFileReferences relMergeFileRefs =
- fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
+ LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
+ .getName(), lastFile.getFile().getName());
ILSMIndexAccessorInternal accessor = new LSMBTreeAccessor(lsmHarness, opCtx);
ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(),
@@ -545,8 +545,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(),
mergeOp.getBloomFilterMergeTarget(), true);
- IIndexBulkLoader bulkLoader =
- mergedComponent.getBTree().createBulkLoader(1.0f, false, numElements, false, true);
+ IIndexBulkLoader bulkLoader = mergedComponent.getBTree().createBulkLoader(1.0f, false, numElements, false);
IIndexBulkLoader builder = null;
if (hasBloomFilter) {
builder = mergedComponent.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(),
@@ -591,20 +590,22 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) factory
.createLSMComponentInstance(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
// BTree will be closed during cleanup of merge().
- if (!createComponent) {
- component.getBTree().activate();
+ if (createComponent) {
+ component.getBTree().create();
}
+ component.getBTree().activate();
if (hasBloomFilter) {
+ if (createComponent) {
+ component.getBloomFilter().create();
+ }
component.getBloomFilter().activate();
}
if (component.getLSMComponentFilter() != null && !createComponent) {
filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getBTree());
}
-
if (!createComponent) {
component.readMostRecentMarkerLSN(component.getBTree());
}
-
return component;
}
@@ -657,12 +658,12 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
throw new TreeIndexException(e);
}
bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput, numElementsHint, false, true);
+ verifyInput, numElementsHint, false);
if (hasBloomFilter) {
int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- BloomFilterSpecification bloomFilterSpec =
- BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
+ bloomFilterFalsePositiveRate);
builder = ((LSMBTreeDiskComponent) component).getBloomFilter().createBuilder(numElementsHint,
bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
} else {
@@ -802,36 +803,36 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@Override
public ITreeIndexFrameFactory getInteriorFrameFactory() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
return mutableComponent.getBTree().getInteriorFrameFactory();
}
@Override
public int getFieldCount() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
return mutableComponent.getBTree().getFieldCount();
}
@Override
public int getFileId() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
return mutableComponent.getBTree().getFileId();
}
@Override
- public IMetaDataPageManager getMetaManager() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getMetaManager();
+ public IPageManager getPageManager() {
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
+ return mutableComponent.getBTree().getPageManager();
}
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
return mutableComponent.getBTree().getLeafFrameFactory();
}
@@ -848,8 +849,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@Override
public int getRootPageId() {
- LSMBTreeMemoryComponent mutableComponent =
- (LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
+ LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents.get(
+ currentMutableComponentId.get());
return mutableComponent.getBTree().getRootPageId();
}
@@ -874,16 +875,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
- try {
- return new LSMBTreeBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
- } catch (HyracksDataException e) {
- throw new TreeIndexException(e);
- }
- }
-
- @Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
Set<String> files = new HashSet<>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index 424dfd3..9768aa3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -21,7 +21,7 @@ package org.apache.hyracks.storage.am.lsm.btree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractDiskLSMComponent;
@@ -65,7 +65,7 @@ public class LSMBTreeDiskComponent extends AbstractDiskLSMComponent {
}
public void readMostRecentMarkerLSN(BTree treeIndex) throws HyracksDataException {
- IMetaDataPageManager treeMetaManager = treeIndex.getMetaManager();
+ IMetadataPageManager treeMetaManager = (IMetadataPageManager) treeIndex.getPageManager();
mostRecentMarkerLSN = treeMetaManager.getLastMarkerLSN();
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 38a3a7b..69c6666 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -30,11 +30,8 @@ import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListMetadataManagerFactory;
import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree;
import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy;
@@ -60,12 +57,12 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeUtils {
public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
- FileReference file,
- IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
- IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
- ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable)
+ FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+ ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+ int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory)
throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
@@ -77,9 +74,6 @@ public class LSMBTreeUtils {
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
- metaFrameFactory);
TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
@@ -115,11 +109,11 @@ public class LSMBTreeUtils {
}
public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
- IBufferCache diskBufferCache,
- IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
- ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- int startWithVersion, boolean durable) {
+ IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, int startWithVersion, boolean durable,
+ IMetadataPageManagerFactory freePageManagerFactory) {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -130,9 +124,6 @@ public class LSMBTreeUtils {
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
- metaFrameFactory);
// This is the tuple writer that can do both inserts and deletes
LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory = new LSMBTreeRefrencingTupleWriterFactory(
typeTraits, cmpFactories.length, false);
@@ -153,12 +144,10 @@ public class LSMBTreeUtils {
TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length);
-
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file,
diskBTreeFactory,
true);
-
// the disk only index uses an empty ArrayList for virtual buffer caches
ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory,
@@ -173,15 +162,13 @@ public class LSMBTreeUtils {
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
- int startWithVersion, boolean durable) {
-
+ int startWithVersion, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
buddyBtreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
buddyBtreeCmpFactories[i] = cmpFactories[buddyBTreeFields[i]];
}
-
TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
buddyBtreeTypeTraits);
ITreeIndexFrameFactory buddyBtreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
@@ -195,10 +182,6 @@ public class LSMBTreeUtils {
ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
- metaFrameFactory);
-
TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
index e97c904..f009db9 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
@@ -44,11 +45,12 @@ public class LSMTreeIndexCompactOperatorDescriptor extends AbstractTreeIndexOper
IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields,
IIndexDataflowHelperFactory dataflowHelperFactory,
- IModificationOperationCallbackFactory modificationOpCallbackProvider) {
+ IModificationOperationCallbackFactory modificationOpCallbackProvider,
+ IPageManagerFactory pageManagerFactory) {
super(spec, 0, 0, null, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
- comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, false,
- false, null,
- NoOpLocalResourceFactoryProvider.INSTANCE, NoOpOperationCallbackFactory.INSTANCE, modificationOpCallbackProvider);
+ comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, false, false, null,
+ NoOpLocalResourceFactoryProvider.INSTANCE, NoOpOperationCallbackFactory.INSTANCE,
+ modificationOpCallbackProvider, pageManagerFactory);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
index 9d1deaf..5403e79 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -31,6 +31,7 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
@@ -53,11 +54,11 @@ public class LSMTreeIndexInsertUpdateDeleteOperatorDescriptor extends AbstractTr
IndexOperation op, IIndexDataflowHelperFactory dataflowHelperFactory,
ITupleFilterFactory tupleFilterFactory, IMissingWriterFactory missingWriterFactory,
IModificationOperationCallbackFactory modificationOpCallbackProvider,
- ISearchOperationCallbackFactory searchOpCallbackProvider) {
+ ISearchOperationCallbackFactory searchOpCallbackProvider, IPageManagerFactory pageManagerFactory) {
super(spec, 1, 1, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, tupleFilterFactory, false, false,
missingWriterFactory, NoOpLocalResourceFactoryProvider.INSTANCE, searchOpCallbackProvider,
- modificationOpCallbackProvider);
+ modificationOpCallbackProvider, pageManagerFactory);
this.fieldPermutation = fieldPermutation;
this.op = op;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
new file mode 100644
index 0000000..5771728
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.freepage;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public class VirtualFreePageManager implements IPageManager {
+ private final AtomicInteger currentPageId = new AtomicInteger();
+ private final IBufferCache bufferCache;
+ private int fileId;
+
+ public VirtualFreePageManager(IBufferCache bufferCache) {
+ // We start the currentPageId from 1, because the BTree uses
+ // the first page as metadata page, and the second page as root page.
+ // (when returning free pages we first increment, then get)
+ this.bufferCache = bufferCache;
+ currentPageId.set(1);
+ }
+
+ @Override
+ public int takePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ // The very first call returns page id 2 because the BTree uses
+ // the first page as metadata page, and the second page as root page.
+ return currentPageId.incrementAndGet();
+ }
+
+ @Override
+ public int takeBlock(ITreeIndexMetaDataFrame metaFrame, int count) throws HyracksDataException {
+ return currentPageId.getAndUpdate(operand -> operand + count) + 1;
+ }
+
+ @Override
+ public int getMaxPageId(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ return currentPageId.get();
+ }
+
+ @Override
+ public ITreeIndexMetaDataFrame createMetadataFrame() {
+ return null;
+ }
+
+ @Override
+ public void releasePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+ throw new HyracksDataException("Pages of an in memory index are released through the virtual buffer cache");
+ }
+
+ @Override
+ public void releaseBlock(ITreeIndexMetaDataFrame metaFrame, int startingPage, int count)
+ throws HyracksDataException {
+ throw new HyracksDataException("Pages of an in memory index are released through the virtual buffer cache");
+ }
+
+ @Override
+ public int getMetadataPageId() {
+ //MD page in a virtual context is always 0, because it is by nature an in-place modification tree
+ return 0;
+ }
+
+ @Override
+ public void open(int fileId) {
+ this.fileId = fileId;
+ }
+
+ @Override
+ public void close() {
+ // Method doesn't make sense for this free page manager.
+ }
+
+ @Override
+ public void init(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory)
+ throws HyracksDataException {
+ currentPageId.set(1);
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, 0), true);
+ page.acquireWriteLatch();
+ page.releaseWriteLatch(false);
+ bufferCache.unpin(page);
+ page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId.get()), true);
+ if (leafFrameFactory != null) {
+ page.acquireWriteLatch();
+ ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
+ leafFrame.setPage(page);
+ leafFrame.initBuffer((byte) 0);
+ page.releaseWriteLatch(true);
+ }
+ bufferCache.unpin(page);
+ }
+
+ @Override
+ public int getRootPageId() throws HyracksDataException {
+ // root index is always 1 for in memory index
+ return 1;
+ }
+
+ @Override
+ public void setRootPageId(int rootPage) throws HyracksDataException {
+ // the root of an in memory index will always be 1
+ }
+
+ @Override
+ public int getBulkLoadLeaf() throws HyracksDataException {
+ return 2;
+ }
+
+ @Override
+ public boolean isEmpty(ITreeIndexFrame frame, int rootPage) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
+ rootNode.acquireReadLatch();
+ try {
+ frame.setPage(rootNode);
+ return frame.getLevel() == 0 && frame.getTupleCount() == 0;
+ } finally {
+ rootNode.releaseReadLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManagerFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManagerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManagerFactory.java
new file mode 100644
index 0000000..0d58b85
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManagerFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.freepage;
+
+import org.apache.hyracks.storage.am.common.api.IPageManager;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public class VirtualFreePageManagerFactory implements IPageManagerFactory {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public IPageManager createPageManager(IBufferCache bufferCache) {
+ return new VirtualFreePageManager(bufferCache);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
deleted file mode 100644
index eab85d2..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common.freepage;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-
-public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager {
- protected final int capacity;
- protected final AtomicInteger currentPageId = new AtomicInteger();
-
- public VirtualMetaDataPageManager(int capacity) {
- // We start the currentPageId from 1, because the BTree uses
- // the first page as metadata page, and the second page as root page.
- // (when returning free pages we first increment, then get)
- currentPageId.set(1);
- this.capacity = capacity;
- }
-
- @Override
- public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- // The very first call returns page id 2 because the BTree uses
- // the first page as metadata page, and the second page as root page.
- return currentPageId.incrementAndGet();
- }
-
- @Override
- public int getFreePageBlock(ITreeIndexMetaDataFrame metaFrame, int count) throws HyracksDataException {
- return currentPageId.getAndUpdate(operand -> operand + count) + 1;
- }
-
- @Override
- public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- return currentPageId.get();
- }
-
- @Override
- public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
- currentPageId.set(1);
- }
-
- @Override
- public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
- return NullMetadataFrameFactory.INSTANCE;
- }
-
- @Override
- public int getCapacity() {
- return capacity - 2;
- }
-
- @Override
- public void reset() {
- currentPageId.set(1);
- }
-
- @Override
- public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
- }
-
- @Override
- public void addFreePageBlock(ITreeIndexMetaDataFrame metaFrame, int startingPage, int count)
- throws HyracksDataException {
- }
-
- @Override
- public byte getMetaPageLevelIndicator() {
- return 0;
- }
-
- @Override
- public byte getFreePageLevelIndicator() {
- return 0;
- }
-
- @Override
- public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
- return false;
- }
-
- @Override
- public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
- return false;
- }
-
- @Override
- public int getFirstMetadataPage() {
- //MD page in a virtual context is always 0, because it is by nature an in-place modification tree
- return 0;
- }
-
- @Override
- public void open(int fileId) {
- // Method doesn't make sense for this free page manager.
- }
-
- @Override
- public void close() {
- // Method doesn't make sense for this free page manager.
- }
-
- private static class NullMetadataFrameFactory implements ITreeIndexMetaDataFrameFactory {
- private static final NullMetadataFrameFactory INSTANCE = new NullMetadataFrameFactory();
-
- @Override
- public ITreeIndexMetaDataFrame createFrame() {
- return null;
- }
-
- }
-
- @Override
- public void init(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- }
-
- @Override
- public int getFilterPageId() throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- return 0;
- }
-
- @Override
- public void setFilterPageId(int filterPageId) throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- }
-
- @Override
- public long getLSN() throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- return -1;
- }
-
- @Override
- public void setLSN(long lsn) throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- }
-
- @Override
- public void setFilterPage(ICachedPage page) {
- // Method doesn't make sense for this free page manager.
- }
-
- @Override
- public ICachedPage getFilterPage() {
- return null;
- }
-
- @Override
- public boolean appendOnlyMode() {
- return false;
- }
-
- @Override
- public long getLSNOffset() throws HyracksDataException {
- return IMetaDataPageManager.INVALID_LSN_OFFSET;
- }
-
- @Override
- public long getLastMarkerLSN() throws HyracksDataException {
- // Method doesn't make sense for this free page manager.
- return -1L;
- }
-
- @Override
- public void setRootPage(int rootPage) throws HyracksDataException {
- // This won't get called for an in-place index. The root page
- // is maintained at a fixed location as in the below method.
- }
-
- @Override
- public int getRootPage() throws HyracksDataException {
- // This also won't be called but the correct answer for an
- // In-place index is always 1.
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 981a2c3..eaca234 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -137,7 +137,7 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
protected void markAsValidInternal(ITreeIndex treeIndex) throws HyracksDataException {
int fileId = treeIndex.getFileId();
IBufferCache bufferCache = treeIndex.getBufferCache();
- treeIndex.getMetaManager().close();
+ treeIndex.getPageManager().close();
// WARNING: flushing the metadata page should be done after releasing the write latch; otherwise, the page
// won't be flushed to disk because it won't be dirty until the write latch has been released.
// Force modified metadata page to disk.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 64742ef..43b6f92 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -90,12 +90,11 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
IBufferCache bufferCache = treeIndex.getBufferCache();
treeIndex.activate();
try {
- int metadataPage = treeIndex.getMetaManager().getFirstMetadataPage();
+ int metadataPage = treeIndex.getPageManager().getMetadataPageId();
if (metadataPage < 0) {
return TreeIndexState.INVALID;
}
- ITreeIndexMetaDataFrame metadataFrame = treeIndex.getMetaManager().getMetaDataFrameFactory()
- .createFrame();
+ ITreeIndexMetaDataFrame metadataFrame = treeIndex.getPageManager().createMetadataFrame();
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage),
false);
page.acquireReadLatch();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
index 9d17b7e..6164c0a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java
@@ -20,11 +20,10 @@
package org.apache.hyracks.storage.am.lsm.common.impls;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -33,7 +32,7 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class BTreeFactory extends TreeIndexFactory<BTree> {
public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ IPageManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) {
super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory,
cmpFactories, fieldCount);
@@ -41,12 +40,8 @@ public class BTreeFactory extends TreeIndexFactory<BTree> {
@Override
public BTree createIndexInstance(FileReference file) throws IndexException {
- try {
- return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createFreePageManager(),
- interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
- } catch (HyracksDataException e) {
- throw new IndexException(e);
- }
+ return new BTree(bufferCache, fileMapProvider, freePageManagerFactory.createPageManager(bufferCache),
+ interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, file);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/bc3d1b05/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
index 7ab70b6..f6d21d8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java
@@ -23,7 +23,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.IIndex;
-import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
@@ -33,10 +33,10 @@ public abstract class IndexFactory<T extends IIndex> {
protected final IIOManager ioManager;
protected final IBufferCache bufferCache;
protected final IFileMapProvider fileMapProvider;
- protected final IMetadataManagerFactory freePageManagerFactory;
+ protected final IPageManagerFactory freePageManagerFactory;
public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider,
- IMetadataManagerFactory freePageManagerFactory) {
+ IPageManagerFactory freePageManagerFactory) {
this.ioManager = ioManager;
this.bufferCache = bufferCache;
this.fileMapProvider = fileMapProvider;