You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by il...@apache.org on 2017/10/16 06:08:06 UTC
[6/6] asterixdb git commit: [NO ISSUE][STO][IDX] LSM storage cleanup
[NO ISSUE][STO][IDX] LSM storage cleanup
- user model changes: no
- storage format changes: no
- interface changes: yes
- Replaced component lifecycle-related fabric methods in AbstractLSMIndex
with direct method calls of lsmComponent functions.
- Extracted common lifecycle-related functionality from index-specific
disk/memory lsmComponents to interfaces.
- Introduced composable disk component bulkloader design which assembles the
proper bulkload pipeline from individual elements populating lsmFilters,
bloomFilters, buddyBTrees\deletedKeysBTrees, bTress\rTrees\invIndexes.
- Changed methods to return index-specific versions of objects (accessors,
components, index instances) to avoid nasty downcasting.
Change-Id: I6739d751b990e7a28e03e32a5de6e2b670d37a1e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2014
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/77f89525
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/77f89525
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/77f89525
Branch: refs/heads/master
Commit: 77f8952573443eaf81270f509ffd39fa84115411
Parents: 536aa3f
Author: Ildar Absalyamov <il...@gmail.com>
Authored: Fri Oct 13 12:22:52 2017 -0700
Committer: Ildar Absalyamov <il...@gmail.com>
Committed: Sun Oct 15 23:06:51 2017 -0700
----------------------------------------------------------------------
asterixdb/asterix-common/pom.xml | 10 +
.../LSMBTreeIOOperationCallback.java | 5 +-
.../LSMBTreeWithBuddyIOOperationCallback.java | 5 +-
.../LSMInvertedIndexIOOperationCallback.java | 2 +-
.../LSMRTreeIOOperationCallback.java | 5 +-
.../LSMSecondaryIndexBulkLoadNodePushable.java | 4 +-
.../hyracks/storage/am/btree/impls/BTree.java | 9 +-
.../am/common/impls/AbstractTreeIndex.java | 1 +
.../am/lsm/btree/impls/ExternalBTree.java | 97 +++----
.../lsm/btree/impls/ExternalBTreeWithBuddy.java | 227 +++------------
.../storage/am/lsm/btree/impls/LSMBTree.java | 242 +++-------------
.../am/lsm/btree/impls/LSMBTreeBulkLoader.java | 62 -----
.../lsm/btree/impls/LSMBTreeDiskComponent.java | 68 ++---
.../impls/LSMBTreeDiskComponentBulkLoader.java | 59 ----
.../impls/LSMBTreeDiskComponentFactory.java | 21 +-
.../impls/LSMBTreeDiskComponentScanCursor.java | 6 +-
.../btree/impls/LSMBTreeMemoryComponent.java | 17 +-
.../am/lsm/btree/impls/LSMBTreeOpContext.java | 14 +-
.../btree/impls/LSMBTreePointSearchCursor.java | 12 +-
.../btree/impls/LSMBTreeRangeSearchCursor.java | 8 +-
.../LSMBTreeWithBloomFilterDiskComponent.java | 86 ++++++
...TreeWithBloomFilterDiskComponentFactory.java | 54 ++++
.../impls/LSMBTreeWithBuddyAbstractCursor.java | 19 +-
.../impls/LSMBTreeWithBuddyDiskComponent.java | 76 +++---
...SMBTreeWithBuddyDiskComponentBulkLoader.java | 65 -----
.../LSMBTreeWithBuddyDiskComponentFactory.java | 28 +-
.../impls/LSMBTreeWithBuddyMemoryComponent.java | 26 +-
.../btree/impls/LSMBuddyBTreeMergeCursor.java | 2 +-
.../am/lsm/btree/utils/LSMBTreeUtil.java | 51 +++-
...AbstractLSMWithBloomFilterDiskComponent.java | 105 +++++++
.../api/AbstractLSMWithBuddyDiskComponent.java | 96 +++++++
.../AbstractLSMWithBuddyMemoryComponent.java | 61 +++++
.../am/lsm/common/api/ILSMComponent.java | 6 +
.../am/lsm/common/api/ILSMDiskComponent.java | 110 +++++++-
.../common/api/ILSMDiskComponentBulkLoader.java | 7 +
.../common/api/ILSMDiskComponentFactory.java | 4 +-
.../storage/am/lsm/common/api/ILSMIndex.java | 17 --
.../am/lsm/common/api/ILSMMemoryComponent.java | 29 +-
.../common/impls/AbstractLSMDiskComponent.java | 89 +++++-
.../AbstractLSMDiskComponentBulkLoader.java | 202 --------------
...ractLSMDiskComponentWithBuddyBulkLoader.java | 156 -----------
.../am/lsm/common/impls/AbstractLSMIndex.java | 126 +++++----
.../impls/AbstractLSMMemoryComponent.java | 46 +++-
.../lsm/common/impls/BloomFilterBulkLoader.java | 67 +++++
.../ChainedLSMDiskComponentBulkLoader.java | 110 ++++++++
.../am/lsm/common/impls/EmptyComponent.java | 67 +++++
.../lsm/common/impls/ExternalIndexHarness.java | 4 +-
.../am/lsm/common/impls/FilterBulkLoader.java | 82 ++++++
.../impls/IChainedComponentBulkLoader.java | 63 +++++
.../common/impls/IndexWithBuddyBulkLoader.java | 72 +++++
.../storage/am/lsm/common/impls/LSMHarness.java | 4 +-
.../am/lsm/common/impls/LSMIndexBulkLoader.java | 67 +++++
.../impls/LSMIndexDiskComponentBulkLoader.java | 64 +++++
.../api/IInPlaceInvertedIndex.java | 9 -
.../invertedindex/impls/LSMInvertedIndex.java | 273 +++----------------
.../impls/LSMInvertedIndexDiskComponent.java | 64 +++--
...LSMInvertedIndexDiskComponentBulkLoader.java | 71 -----
.../LSMInvertedIndexDiskComponentFactory.java | 6 +-
.../impls/LSMInvertedIndexMemoryComponent.java | 32 +--
.../impls/LSMInvertedIndexOpContext.java | 14 +-
.../impls/PartitionedLSMInvertedIndex.java | 32 +--
.../inmemory/InMemoryInvertedIndex.java | 6 +-
.../inmemory/InMemoryInvertedIndexAccessor.java | 6 +-
.../InMemoryInvertedIndexOpContext.java | 4 +-
.../PartitionedInMemoryInvertedIndex.java | 6 +-
.../ondisk/OnDiskInvertedIndex.java | 4 +-
.../ondisk/OnDiskInvertedIndexFactory.java | 3 +-
.../ondisk/PartitionedOnDiskInvertedIndex.java | 3 +-
.../PartitionedOnDiskInvertedIndexFactory.java | 3 +-
.../invertedindex/util/InvertedIndexUtils.java | 26 +-
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 117 ++------
.../am/lsm/rtree/impls/ExternalRTree.java | 111 +++-----
.../storage/am/lsm/rtree/impls/LSMRTree.java | 180 +++---------
.../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 16 +-
.../am/lsm/rtree/impls/LSMRTreeBulkLoader.java | 60 ----
.../LSMRTreeDeletedKeysBTreeMergeCursor.java | 2 +-
.../lsm/rtree/impls/LSMRTreeDiskComponent.java | 89 +++---
.../impls/LSMRTreeDiskComponentBulkLoader.java | 65 -----
.../impls/LSMRTreeDiskComponentFactory.java | 6 +-
.../rtree/impls/LSMRTreeMemoryComponent.java | 24 +-
.../am/lsm/rtree/impls/LSMRTreeOpContext.java | 12 +-
.../impls/LSMRTreeWithAntiMatterTuples.java | 155 ++---------
...AntiMatterTuplesDiskComponentBulkLoader.java | 59 ----
...ithAntiMatterTuplesDiskComponentFactory.java | 9 +-
...SMRTreeWithAntiMatterTuplesSearchCursor.java | 6 +-
.../LSMRTreeWithAntimatterDiskComponent.java | 71 +++++
.../am/lsm/rtree/utils/LSMRTreeUtils.java | 43 ++-
.../hyracks/storage/am/rtree/impls/RTree.java | 6 +-
.../apache/hyracks/storage/common/IIndex.java | 11 +
.../storage/am/common/IndexTestContext.java | 3 +-
.../am/lsm/btree/LSMBTreeFileManagerTest.java | 7 +-
...TreeUpdateInPlaceScanDiskComponentsTest.java | 12 +-
.../storage/am/lsm/btree/impl/TestLsmBtree.java | 13 +-
.../am/lsm/btree/impl/TestLsmBtreeUtil.java | 21 +-
94 files changed, 2156 insertions(+), 2439 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 6df7049..42a8004 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -168,6 +168,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+ <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-btree:*</ignoredUnusedDeclaredDependency>
+ <ignoredUnusedDeclaredDependency>org.apache.hyracks:hyracks-storage-am-lsm-rtree:*</ignoredUnusedDeclaredDependency>
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </plugin>
</plugins>
</build>
<dependencies>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
index 8c91879..147bc10 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeIOOperationCallback.java
@@ -20,8 +20,8 @@
package org.apache.asterix.common.ioopcallbacks;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
-import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
@@ -35,9 +35,8 @@ public class LSMBTreeIOOperationCallback extends AbstractLSMIOOperationCallback
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeFileManager.BTREE_SUFFIX)) {
- LSMBTreeDiskComponent btreeComponent = (LSMBTreeDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
- (IMetadataPageManager) btreeComponent.getBTree().getPageManager();
+ (IMetadataPageManager) ((BTree) diskComponent.getIndex()).getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
index 9f50f57..5b6de6c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallback.java
@@ -19,8 +19,8 @@
package org.apache.asterix.common.ioopcallbacks;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
-import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponent;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
@@ -30,9 +30,8 @@ public class LSMBTreeWithBuddyIOOperationCallback extends AbstractLSMIOOperation
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMBTreeWithBuddyFileManager.BTREE_SUFFIX)) {
- LSMBTreeWithBuddyDiskComponent btreeComponent = (LSMBTreeWithBuddyDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
- (IMetadataPageManager) btreeComponent.getBTree().getPageManager();
+ (IMetadataPageManager) ((BTree) diskComponent.getIndex()).getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
index 69286d5..3c91a6a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallback.java
@@ -37,7 +37,7 @@ public class LSMInvertedIndexIOOperationCallback extends AbstractLSMIOOperationC
if (diskComponentFilePath.endsWith(LSMInvertedIndexFileManager.DELETED_KEYS_BTREE_SUFFIX)) {
LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
- (IMetadataPageManager) invIndexComponent.getDeletedKeysBTree().getPageManager();
+ (IMetadataPageManager) invIndexComponent.getBuddyIndex().getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
index 1001af9..063fe38 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallback.java
@@ -22,8 +22,8 @@ package org.apache.asterix.common.ioopcallbacks;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeDiskComponent;
import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager;
+import org.apache.hyracks.storage.am.rtree.impls.RTree;
public class LSMRTreeIOOperationCallback extends AbstractLSMIOOperationCallback {
@@ -35,9 +35,8 @@ public class LSMRTreeIOOperationCallback extends AbstractLSMIOOperationCallback
public long getComponentFileLSNOffset(ILSMDiskComponent diskComponent, String diskComponentFilePath)
throws HyracksDataException {
if (diskComponentFilePath.endsWith(LSMRTreeFileManager.RTREE_SUFFIX)) {
- LSMRTreeDiskComponent rtreeComponent = (LSMRTreeDiskComponent) diskComponent;
IMetadataPageManager metadataPageManager =
- (IMetadataPageManager) rtreeComponent.getRTree().getPageManager();
+ (IMetadataPageManager) ((RTree) diskComponent.getIndex()).getPageManager();
return metadataPageManager.getFileOffset(metadataPageManager.createMetadataFrame(), LSN_KEY);
}
return INVALID;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
index 12f49a8..cb81b64 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryIndexBulkLoadNodePushable.java
@@ -199,8 +199,8 @@ public class LSMSecondaryIndexBulkLoadNodePushable extends AbstractLSMSecondaryI
endCurrentComponent();
component = secondaryIndex.createBulkLoadTarget();
- componentBulkLoader = secondaryIndex.createComponentBulkLoader(component, 1.0f, false,
- getNumDeletedTuples(componentPos), false, true, true);
+ int numTuples = getNumDeletedTuples(componentPos);
+ componentBulkLoader = component.createBulkLoader(1.0f, false, numTuples, false, true, true);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 7206fee..5e3042e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -118,8 +118,7 @@ public class BTree extends AbstractTreeIndex {
// Stack validation protocol:
// * parent pushes the validation information onto the stack before validation
// * child pops the validation information off of the stack after validating
- BTreeAccessor accessor =
- (BTreeAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ BTreeAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
PageValidationInfo pvi = accessor.ctx.createPageValidationInfo(null);
accessor.ctx.getValidationInfos().addFirst(pvi);
if (isActive) {
@@ -819,12 +818,12 @@ public class BTree extends AbstractTreeIndex {
}
@Override
- public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ public BTreeAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
return new BTreeAccessor(this, modificationCallback, searchCallback);
}
- public ITreeIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
+ public BTreeAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback, int[] logTupleFields) {
return new BTreeAccessor(this, modificationCallback, searchCallback, logTupleFields);
}
@@ -892,7 +891,7 @@ public class BTree extends AbstractTreeIndex {
}
@Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
+ public BTreeRangeSearchCursor createSearchCursor(boolean exclusive) {
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
return new BTreeRangeSearchCursor(leafFrame, exclusive);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 8ffd0eb..ae66402 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -121,6 +121,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
isActive = false;
}
+ @Override
public void purge() throws HyracksDataException {
if (isActive) {
throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 3775985..24f513a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -28,8 +28,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
@@ -41,6 +39,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -56,12 +55,12 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
/**
* This is an lsm b-tree that does not have memory component and is modified
@@ -76,7 +75,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
// This component factory has to be different since it uses different tuple
// writer in it's leaf frames to support inserting both
// regular and delete tuples
- private final LSMBTreeDiskComponentFactory transactionComponentFactory;
+ private final ILSMDiskComponentFactory transactionComponentFactory;
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
private final List<ILSMDiskComponent> secondDiskComponents;
@@ -90,16 +89,15 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
public ExternalBTree(IIOManager ioManager, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
- ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
- TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
+ IBufferCache bufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ ILSMDiskComponentFactory bulkLoadComponentFactory, ILSMDiskComponentFactory transactionComponentFactory,
double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- TreeIndexFactory<BTree> transactionBTreeFactory, boolean durable) {
- super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
- bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, false, durable);
- this.transactionComponentFactory =
- new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
+ boolean durable) {
+ super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, bufferCache, fileManager, componentFactory,
+ bulkLoadComponentFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker,
+ ioScheduler, ioOpCallback, false, durable);
+ this.transactionComponentFactory = transactionComponentFactory;
this.secondDiskComponents = new LinkedList<>();
this.interiorFrameFactory = interiorFrameFactory;
}
@@ -164,7 +162,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
// This method creates the appropriate opContext for the targeted version
public ExternalBTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
return new ExternalBTreeOpContext(insertLeafFrameFactory, deleteLeafFrameFactory, searchCallback,
- componentFactory.getBloomFilterKeyFields().length, cmpFactories, targetVersion, getLsmHarness());
+ ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length,
+ cmpFactories, targetVersion, getLsmHarness());
}
// The only reason to override the following method is that it uses a different context object
@@ -188,8 +187,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
}
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
- BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
- BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
+ BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getIndex();
+ BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getIndex();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
LSMComponentFileReferences relMergeFileRefs =
@@ -252,32 +251,23 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
//First time activation
List<LSMComponentFileReferences> validFileReferences;
validFileReferences = fileManager.cleanupAndGetValidFiles();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMBTreeDiskComponent component;
- component =
- createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
+ for (LSMComponentFileReferences lsmComponentFileReferences : validFileReferences) {
+ ILSMDiskComponent component =
+ createDiskComponent(componentFactory, lsmComponentFileReferences.getInsertIndexFileReference(),
+ null, lsmComponentFileReferences.getBloomFilterFileReference(), false);
diskComponents.add(component);
secondDiskComponents.add(component);
}
getLsmHarness().indexFirstTimeActivated();
} else {
// This index has been opened before
- for (ILSMComponent c : diskComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- BTree btree = component.getBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.activate();
- bloomFilter.activate();
+ for (ILSMDiskComponent c : diskComponents) {
+ c.activate(false);
}
- for (ILSMComponent c : secondDiskComponents) {
+ for (ILSMDiskComponent c : secondDiskComponents) {
// Only activate non shared components
if (!diskComponents.contains(c)) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- BTree btree = component.getBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.activate();
- bloomFilter.activate();
+ c.activate(false);
}
}
}
@@ -300,12 +290,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
cb.afterFinalize(LSMOperationType.FLUSH, null);
}
for (ILSMDiskComponent c : diskComponents) {
- deactivateDiskComponent(c);
+ c.deactivateAndPurge();
}
for (ILSMDiskComponent c : secondDiskComponents) {
// Only deactivate non shared components
if (!diskComponents.contains(c)) {
- deactivateDiskComponent(c);
+ c.deactivateAndPurge();
}
}
isActive = false;
@@ -320,22 +310,14 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
getLsmHarness().indexClear();
- for (ILSMComponent c : diskComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBloomFilter().deactivate();
- component.getBTree().deactivate();
- component.getBloomFilter().destroy();
- component.getBTree().destroy();
+ for (ILSMDiskComponent c : diskComponents) {
+ c.deactivateAndDestroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
- for (ILSMComponent c : secondDiskComponents) {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBloomFilter().deactivate();
- component.getBTree().deactivate();
- component.getBloomFilter().destroy();
- component.getBTree().destroy();
+ for (ILSMDiskComponent c : secondDiskComponents) {
+ c.deactivateAndDestroy();
}
diskComponents.clear();
secondDiskComponents.clear();
@@ -348,12 +330,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
for (ILSMDiskComponent c : diskComponents) {
- destroyDiskComponent(c);
+ c.destroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
for (ILSMDiskComponent c : secondDiskComponents) {
- destroyDiskComponent(c);
+ c.destroy();
}
diskComponents.clear();
secondDiskComponents.clear();
@@ -448,7 +430,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
componentBulkLoader =
- createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, true);
+ component.createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true, true);
}
// It is expected that the mode was set to insert operation before
@@ -466,10 +448,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
// Since this is a transaction component, validate and
// deactivate. it could later be added or deleted
component.markAsValid(durable);
- BTree btree = ((LSMBTreeDiskComponent) component).getBTree();
- BloomFilter bloomFilter = ((LSMBTreeDiskComponent) component).getBloomFilter();
- btree.deactivate();
- bloomFilter.deactivate();
+ component.deactivate();
} else {
getLsmHarness().addBulkLoadedComponent(component);
}
@@ -502,7 +481,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
throw HyracksDataException.create(e);
}
return createDiskComponent(transactionComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), true);
+ null, componentFileRefs.getBloomFilterFileReference(), true);
}
}
@@ -568,11 +547,11 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
@Override
public void commitTransaction() throws HyracksDataException {
- LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
- LSMBTreeDiskComponent component = null;
- if (componentFileRefrences != null) {
- component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
- componentFileRefrences.getBloomFilterFileReference(), false);
+ LSMComponentFileReferences componentFileReferences = fileManager.getTransactionFileReferenceForCommit();
+ ILSMDiskComponent component = null;
+ if (componentFileReferences != null) {
+ component = createDiskComponent(componentFactory, componentFileReferences.getInsertIndexFileReference(),
+ null, componentFileReferences.getBloomFilterFileReference(), false);
}
getLsmHarness().addTransactionComponents(component);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index ff17905..55090ad 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -19,10 +19,8 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
import java.io.IOException;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -30,10 +28,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
@@ -44,6 +38,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
@@ -54,7 +49,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
@@ -64,22 +58,15 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationC
import org.apache.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex {
- // For creating merge disk components
- private final LSMBTreeWithBuddyDiskComponentFactory componentFactory;
-
- private final LSMBTreeWithBuddyDiskComponentFactory bulkComponentFactory;
-
private final IBinaryComparatorFactory[] btreeCmpFactories;
private final IBinaryComparatorFactory[] buddyBtreeCmpFactories;
private final int[] buddyBTreeFields;
@@ -96,25 +83,19 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
public ExternalBTreeWithBuddy(IIOManager ioManager, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory,
- IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
- TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
- TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory,
- double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
- IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
- int[] buddyBTreeFields, boolean durable) {
+ IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ ILSMDiskComponentFactory bulkLoadComponentFactory, double bloomFilterFalsePositiveRate,
+ ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallback ioOpCallback, IBinaryComparatorFactory[] btreeCmpFactories,
+ IBinaryComparatorFactory[] buddyBtreeCmpFactories, int[] buddyBTreeFields, boolean durable) {
super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback, durable);
+ ioScheduler, ioOpCallback, componentFactory, bulkLoadComponentFactory, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
this.buddyBTreeFields = buddyBTreeFields;
this.btreeInteriorFrameFactory = btreeInteriorFrameFactory;
this.btreeLeafFrameFactory = btreeLeafFrameFactory;
this.buddyBtreeLeafFrameFactory = buddyBtreeLeafFrameFactory;
- this.componentFactory =
- new LSMBTreeWithBuddyDiskComponentFactory(copyBtreeFactory, buddyBtreeFactory, bloomFilterFactory);
- this.bulkComponentFactory =
- new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory, bloomFilterFactory);
this.secondDiskComponents = new LinkedList<>();
}
@@ -125,22 +106,16 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
- return null;
- }
-
- @Override
public void activate() throws HyracksDataException {
if (isActive) {
throw new HyracksDataException("Failed to activate the index since it is already activated.");
}
-
if (diskComponents.size() == 0 && secondDiskComponents.size() == 0) {
//First time activation
List<LSMComponentFileReferences> validFileReferences;
validFileReferences = fileManager.cleanupAndGetValidFiles();
for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMBTreeWithBuddyDiskComponent component;
+ ILSMDiskComponent component;
component =
createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
lsmComonentFileReference.getDeleteIndexFileReference(),
@@ -152,25 +127,13 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
} else {
// This index has been opened before or is brand new with no
// components. It should also maintain the version pointer
- for (ILSMComponent c : diskComponents) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- BTree btree = component.getBTree();
- BTree buddyBtree = component.getBuddyBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.activate();
- buddyBtree.activate();
- bloomFilter.activate();
+ for (ILSMDiskComponent c : diskComponents) {
+ c.activate(false);
}
- for (ILSMComponent c : secondDiskComponents) {
+ for (ILSMDiskComponent c : secondDiskComponents) {
// Only activate non shared components
if (!diskComponents.contains(c)) {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- BTree btree = component.getBTree();
- BTree buddyBtree = component.getBuddyBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.activate();
- buddyBtree.activate();
- bloomFilter.activate();
+ c.activate(false);
}
}
}
@@ -184,12 +147,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
((ExternalIndexHarness) getLsmHarness()).indexClear();
for (ILSMDiskComponent c : diskComponents) {
- clearDiskComponent(c);
+ c.deactivateAndDestroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
for (ILSMDiskComponent c : secondDiskComponents) {
- clearDiskComponent(c);
+ c.deactivateAndDestroy();
}
diskComponents.clear();
secondDiskComponents.clear();
@@ -197,28 +160,17 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().deactivate();
- component.getBuddyBTree().deactivate();
- component.getBloomFilter().deactivate();
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getBuddyBTree().destroy();
- }
-
- @Override
public void destroy() throws HyracksDataException {
if (isActive) {
throw new HyracksDataException("Failed to destroy the index since it is activated.");
}
for (ILSMDiskComponent c : diskComponents) {
- destroyDiskComponent(c);
+ c.destroy();
// Remove from second list to avoid destroying twice
secondDiskComponents.remove(c);
}
for (ILSMDiskComponent c : secondDiskComponents) {
- destroyDiskComponent(c);
+ c.destroy();
}
diskComponents.clear();
secondDiskComponents.clear();
@@ -227,14 +179,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- component.getBTree().destroy();
- component.getBuddyBTree().destroy();
- component.getBloomFilter().destroy();
- }
-
- @Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) throws HyracksDataException {
return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, version),
@@ -269,27 +213,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
newerList.add(swapIndex, newComponent);
}
- @Override
- public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
- boolean cleanupEmptyComponent) throws HyracksDataException {
- BloomFilterSpecification bloomFilterSpec = null;
- if (numElementsHint > 0) {
- int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
- }
- if (withFilter && filterFields != null) {
- return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component,
- bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent,
- filterManager, treeFields, filterFields,
- MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories()));
- } else {
- return new LSMBTreeWithBuddyDiskComponentBulkLoader((LSMBTreeWithBuddyDiskComponent) component,
- bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- cleanupEmptyComponent);
- }
- }
-
// For initial load
@Override
public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
@@ -331,9 +254,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
throws HyracksDataException {
- BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getBTree();
+ BTree firstTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(0)).getIndex();
BTree lastTree = ((LSMBTreeWithBuddyDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1))
- .getBTree();
+ .getIndex();
FileReference firstFile = firstTree.getFileReference();
FileReference lastFile = lastTree.getFileReference();
LSMComponentFileReferences fileRefs =
@@ -385,10 +308,10 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
ILSMIndexOperationContext opCtx = ((LSMBTreeWithBuddySortedCursor) cursor).getOpCtx();
search(opCtx, cursor, btreeSearchPred);
- LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ ILSMDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
mergeOp.getBuddyBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
- IIndexBulkLoader componentBulkLoader;
+ ILSMDiskComponentBulkLoader componentBulkLoader;
// In case we must keep the deleted-keys BuddyBTrees, then they must be
// merged *before* merging the b-trees so that
@@ -403,24 +326,23 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
- numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter()
- .getNumElements();
+ numElements += ((AbstractLSMWithBloomFilterDiskComponent) mergeOp.getMergingComponents().get(i))
+ .getBloomFilter().getNumElements();
}
- componentBulkLoader =
- createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false);
+ componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false);
try {
while (buddyBtreeCursor.hasNext()) {
buddyBtreeCursor.next();
ITupleReference tuple = buddyBtreeCursor.getTuple();
- ((LSMBTreeWithBuddyDiskComponentBulkLoader) componentBulkLoader).delete(tuple);
+ componentBulkLoader.delete(tuple);
}
} finally {
buddyBtreeCursor.close();
}
} else {
- componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false, false);
+ componentBulkLoader = mergedComponent.createBulkLoader(1.0f, false, 0L, false, false, false);
}
try {
@@ -515,12 +437,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// modify any of the lists to make sure they
// are there if the index was opened again
for (ILSMDiskComponent c : diskComponents) {
- deactivateDiskComponent(c);
+ c.deactivateAndPurge();
}
for (ILSMDiskComponent c : secondDiskComponents) {
// Only deactivate non shared components
if (!diskComponents.contains(c)) {
- deactivateDiskComponent(c);
+ c.deactivateAndPurge();
}
}
isActive = false;
@@ -564,23 +486,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
return btreeCmpFactories;
}
- private LSMBTreeWithBuddyDiskComponent createDiskComponent(ILSMDiskComponentFactory factory,
- FileReference insertFileRef, FileReference deleteFileRef, FileReference bloomFilterFileRef,
- boolean createComponent) throws HyracksDataException {
- // Create new instance.
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) factory
- .createComponent(new LSMComponentFileReferences(insertFileRef, deleteFileRef, bloomFilterFileRef));
- if (createComponent) {
- component.getBTree().create();
- component.getBuddyBTree().create();
- component.getBloomFilter().create();
- }
- component.getBTree().activate();
- component.getBuddyBTree().activate();
- component.getBloomFilter().activate();
- return component;
- }
-
// even though the index doesn't support record level modification, the
// accessor will try to do it
// we could throw the exception here but we don't. it will eventually be
@@ -590,7 +495,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// modifications
public class LSMTwoPCBTreeWithBuddyBulkLoader implements IIndexBulkLoader, ITwoPCIndexBulkLoader {
private final ILSMDiskComponent component;
- private final IIndexBulkLoader componentBulkLoader;
+ private final ILSMDiskComponentBulkLoader componentBulkLoader;
private final boolean isTransaction;
public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
@@ -604,7 +509,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
componentBulkLoader =
- createComponentBulkLoader(component, fillFactor, verifyInput, numElementsHint, false, true, false);
+ component.createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true, false);
}
@Override
@@ -620,12 +525,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// Since this is a transaction component, validate and
// deactivate. it could later be added or deleted
component.markAsValid(durable);
- BTree btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree();
- BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
- BloomFilter bloomFilter = ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter();
- btree.deactivate();
- buddyBtree.deactivate();
- bloomFilter.deactivate();
+ component.deactivate();
} else {
getLsmHarness().addBulkLoadedComponent(component);
}
@@ -634,7 +534,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
@Override
public void delete(ITupleReference tuple) throws HyracksDataException {
- ((LSMBTreeWithBuddyDiskComponentBulkLoader) componentBulkLoader).delete(tuple);
+ componentBulkLoader.delete(tuple);
}
@Override
@@ -654,20 +554,13 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
} catch (IOException e) {
throw HyracksDataException.create(e);
}
- return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
+ return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
true);
}
}
@Override
- public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return createDiskComponent(bulkComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), true);
- }
-
- @Override
public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
throws HyracksDataException {
return new LSMTreeIndexAccessor(getLsmHarness(), createOpContext(searchCallback, targetIndexVersion),
@@ -709,7 +602,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
@Override
public void commitTransaction() throws HyracksDataException {
LSMComponentFileReferences componentFileRefrences = fileManager.getTransactionFileReferenceForCommit();
- LSMBTreeWithBuddyDiskComponent component = null;
+ ILSMDiskComponent component = null;
if (componentFileRefrences != null) {
component = createDiskComponent(componentFactory, componentFileRefrences.getInsertIndexFileReference(),
componentFileRefrences.getDeleteIndexFileReference(),
@@ -739,60 +632,6 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
@Override
- public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<>();
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
- files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
- files.add(component.getBuddyBTree().getFileReference().getFile().getAbsolutePath());
- files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath());
- return files;
- }
-
- @Override
- protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- //do nothing since external index never use memory components
- }
-
- @Override
- protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) c;
- BTree btree = component.getBTree();
- BTree buddyBtree = component.getBuddyBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- btree.deactivate();
- buddyBtree.deactivate();
- bloomFilter.deactivate();
- btree.purge();
- buddyBtree.purge();
- bloomFilter.purge();
- }
-
- @Override
- protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- //do nothing since external index never use memory components
- }
-
- @Override
- protected long getMemoryComponentSize(ILSMMemoryComponent c) {
- return 0;
- }
-
- @Override
- protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
- }
-
- @Override
- protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
- }
-
- @Override
- protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- //do nothing since external index never use memory components
- }
-
- @Override
protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
ILSMDiskComponent lastComponent) throws HyracksDataException {
return null;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index c7d45e1..f6942b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -20,9 +20,7 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -31,9 +29,6 @@ import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
@@ -44,18 +39,19 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
@@ -66,9 +62,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.common.IIndexAccessor;
-import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
@@ -80,12 +74,6 @@ import org.apache.hyracks.util.trace.ITracer;
public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
private static final ICursorFactory cursorFactory = opCtx -> new LSMBTreeSearchCursor(opCtx);
- // For creating BTree's used in flush and merge.
- protected final LSMBTreeDiskComponentFactory componentFactory;
- // For creating BTree's used in bulk load. Different from diskBTreeFactory
- // because it should have a different tuple writer in it's leaf frames.
- protected final LSMBTreeDiskComponentFactory bulkLoadComponentFactory;
-
// Common for in-memory and on-disk components.
protected final ITreeIndexFrameFactory insertLeafFrameFactory;
protected final ITreeIndexFrameFactory deleteLeafFrameFactory;
@@ -99,16 +87,16 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
- ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
- TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
- BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper,
+ ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache diskBufferCache,
+ ILSMIndexFileManager fileManager, ILSMDiskComponentFactory componentFactory,
+ ILSMDiskComponentFactory bulkLoadComponentFactory, IComponentFilterHelper filterHelper,
ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
boolean durable, boolean updateAware, ITracer tracer) throws HyracksDataException {
- super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
+ super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker, ioScheduler, ioOpCallback, componentFactory, bulkLoadComponentFactory, filterFrameFactory,
filterManager, filterFields, durable, filterHelper, btreeFields, tracer);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
@@ -125,35 +113,25 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
memoryComponents.add(mutableComponent);
++i;
}
- componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterHelper);
- bulkLoadComponentFactory =
- new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, filterHelper);
this.needKeyDupCheck = needKeyDupCheck;
this.hasBloomFilter = needKeyDupCheck;
}
// Without memory components
public LSMBTree(IIOManager ioManager, ITreeIndexFrameFactory insertLeafFrameFactory,
- ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
- TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
- BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
- IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
- ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
- boolean durable) {
- super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy,
- opTracker, ioScheduler, ioOpCallback, durable);
+ ITreeIndexFrameFactory deleteLeafFrameFactory, IBufferCache bufferCache, ILSMIndexFileManager fileManager,
+ ILSMDiskComponentFactory componentFactory, ILSMDiskComponentFactory bulkLoadComponentFactory,
+ double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+ boolean needKeyDupCheck, boolean durable) {
+ super(ioManager, bufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, componentFactory, bulkLoadComponentFactory, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
this.needKeyDupCheck = needKeyDupCheck;
this.hasBloomFilter = true;
this.updateAware = false;
- componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
- bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
- }
-
- public boolean hasBloomFilter() {
- return hasBloomFilter;
}
@Override
@@ -167,35 +145,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
@Override
- protected ILSMDiskComponent loadComponent(LSMComponentFileReferences lsmComonentFileReferences)
- throws HyracksDataException {
- return createDiskComponent(componentFactory, lsmComonentFileReferences.getInsertIndexFileReference(),
- lsmComonentFileReferences.getBloomFilterFileReference(), false);
- }
-
- @Override
- protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBTree().destroy();
- if (hasBloomFilter) {
- component.getBloomFilter().destroy();
- }
- }
-
- @Override
- protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- if (hasBloomFilter) {
- component.getBloomFilter().deactivate();
- }
- component.getBTree().deactivate();
- if (hasBloomFilter) {
- component.getBloomFilter().destroy();
- }
- component.getBTree().destroy();
- }
-
- @Override
public void modify(IIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException {
LSMBTreeOpContext ctx = (LSMBTreeOpContext) ictx;
ITupleReference indexTuple;
@@ -297,7 +246,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
public ILSMDiskComponent doFlush(ILSMIOOperation operation) throws HyracksDataException {
LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
- IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE,
+ IIndexAccessor accessor = flushingComponent.getIndex().createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
@@ -317,11 +266,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
}
- LSMBTreeDiskComponent component =
- createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
+ ILSMDiskComponent component =
+ createDiskComponent(componentFactory, flushOp.getTarget(), null, flushOp.getBloomFilterTarget(), true);
ILSMDiskComponentBulkLoader componentBulkLoader =
- createComponentBulkLoader(component, 1.0f, false, numElements, false, false, false);
+ component.createBulkLoader(1.0f, false, numElements, false, false, false);
IIndexCursor scanCursor = accessor.createSearchCursor(false);
accessor.search(scanCursor, nullPred);
@@ -344,7 +293,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
- getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
}
// Write metadata from memory component to disk
// Q. what about the merge operation? how do we resolve conflicts
@@ -370,14 +319,15 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
if (hasBloomFilter) {
//count elements in btree for creating Bloomfilter
for (int i = 0; i < mergedComponents.size(); ++i) {
- numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
+ numElements += ((AbstractLSMWithBloomFilterDiskComponent) mergedComponents.get(i)).getBloomFilter()
+ .getNumElements();
}
}
- LSMBTreeDiskComponent mergedComponent =
- createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
+ ILSMDiskComponent mergedComponent =
+ createDiskComponent(componentFactory, mergeOp.getTarget(), null, mergeOp.getBloomFilterTarget(), true);
ILSMDiskComponentBulkLoader componentBulkLoader =
- createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false, false);
+ mergedComponent.createBulkLoader(1.0f, false, numElements, false, false, false);
try {
while (cursor.hasNext()) {
cursor.next();
@@ -394,69 +344,13 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
- getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
+ getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(),
+ mergedComponent.getMetadataHolder());
}
componentBulkLoader.end();
return mergedComponent;
}
- protected LSMBTreeDiskComponent createDiskComponent(LSMBTreeDiskComponentFactory factory,
- FileReference btreeFileRef, FileReference bloomFilterFileRef, boolean createComponent)
- throws HyracksDataException {
- // Create new BTree instance.
- LSMBTreeDiskComponent component =
- factory.createComponent(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
- // BTree will be closed during cleanup of merge().
- if (createComponent) {
- component.getBTree().create();
- }
- component.getBTree().activate();
- if (hasBloomFilter) {
- if (createComponent) {
- component.getBloomFilter().create();
- }
- component.getBloomFilter().activate();
- }
- if (component.getLSMComponentFilter() != null && !createComponent) {
- getFilterManager().readFilter(component.getLSMComponentFilter(), component.getBTree());
- }
- return component;
- }
-
- @Override
- public ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
- boolean cleanupEmptyComponent) throws HyracksDataException {
- BloomFilterSpecification bloomFilterSpec = null;
- if (hasBloomFilter) {
- int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
- bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
- }
-
- if (withFilter && filterFields != null) {
- return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor,
- verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent, filterManager, treeFields,
- filterFields, MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories()));
- } else {
- return new LSMBTreeDiskComponentBulkLoader((LSMBTreeDiskComponent) component, bloomFilterSpec, fillFactor,
- verifyInput, numElementsHint, checkIfEmptyIndex, cleanupEmptyComponent);
- }
-
- }
-
- @Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
- throws HyracksDataException {
- return new LSMBTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint);
- }
-
- @Override
- public ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- return createDiskComponent(bulkLoadComponentFactory, componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), true);
- }
-
@Override
protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
LSMComponentFileReferences componentFileRefs, ILSMIOOperationCallback callback) {
@@ -468,7 +362,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@Override
public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- int numBloomFilterKeyFields = hasBloomFilter ? componentFactory.getBloomFilterKeyFields().length : 0;
+ int numBloomFilterKeyFields = hasBloomFilter
+ ? ((LSMBTreeWithBloomFilterDiskComponentFactory) componentFactory).getBloomFilterKeyFields().length
+ : 0;
return new LSMBTreeOpContext(memoryComponents, insertLeafFrameFactory, deleteLeafFrameFactory,
modificationCallback, searchCallback, numBloomFilterKeyFields, getTreeFields(), getFilterFields(),
getLsmHarness(), getFilterCmpFactories());
@@ -488,113 +384,49 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
public ITreeIndexFrameFactory getInteriorFrameFactory() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getInteriorFrameFactory();
+ return mutableComponent.getIndex().getInteriorFrameFactory();
}
@Override
public int getFieldCount() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getFieldCount();
+ return mutableComponent.getIndex().getFieldCount();
}
@Override
public int getFileId() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getFileId();
+ return mutableComponent.getIndex().getFileId();
}
@Override
public IPageManager getPageManager() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getPageManager();
+ return mutableComponent.getIndex().getPageManager();
}
@Override
public ITreeIndexFrameFactory getLeafFrameFactory() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getLeafFrameFactory();
+ return mutableComponent.getIndex().getLeafFrameFactory();
}
@Override
public int getRootPageId() {
LSMBTreeMemoryComponent mutableComponent =
(LSMBTreeMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
- return mutableComponent.getBTree().getRootPageId();
- }
-
- @Override
- protected long getMemoryComponentSize(ILSMMemoryComponent c) {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- IBufferCache virtualBufferCache = mutableComponent.getBTree().getBufferCache();
- return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize();
- }
-
- @Override
- public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<>();
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
- files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
- if (hasBloomFilter) {
- files.add(component.getBloomFilter().getFileReference().getFile().getAbsolutePath());
- }
- return files;
- }
-
- @Override
- protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().clear();
- mutableComponent.reset();
- }
-
- @Override
- protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().validate();
- }
-
- @Override
- protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
- btree.validate();
- }
-
- @Override
- protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
- LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
- component.getBTree().deactivate();
- component.getBTree().purge();
- if (hasBloomFilter) {
- component.getBloomFilter().deactivate();
- component.getBloomFilter().purge();
- }
- }
-
- @Override
- protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- mutableComponent.getBTree().deactivate();
- mutableComponent.getBTree().destroy();
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
- }
-
- @Override
- protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
- LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
- ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
- mutableComponent.getBTree().create();
- mutableComponent.getBTree().activate();
+ return mutableComponent.getIndex().getRootPageId();
}
@Override
protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
ILSMDiskComponent lastComponent) throws HyracksDataException {
- BTree firstBTree = ((LSMBTreeDiskComponent) firstComponent).getBTree();
- BTree lastBTree = ((LSMBTreeDiskComponent) lastComponent).getBTree();
+ BTree firstBTree = (BTree) firstComponent.getIndex();
+ BTree lastBTree = (BTree) lastComponent.getIndex();
FileReference firstFile = firstBTree.getFileReference();
FileReference lastFile = lastBTree.getFileReference();
return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());