You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by il...@apache.org on 2017/10/16 06:08:05 UTC
[5/6] asterixdb git commit: [NO ISSUE][STO][IDX] LSM storage cleanup
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java
deleted file mode 100644
index c169d85..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeBulkLoader.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.btree.impls;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-import org.apache.hyracks.storage.common.IIndexBulkLoader;
-
-public class LSMBTreeBulkLoader implements IIndexBulkLoader {
- private final LSMBTree lsmIndex;
- private final ILSMDiskComponent component;
- private final IIndexBulkLoader componentBulkLoader;
-
- public LSMBTreeBulkLoader(LSMBTree lsmIndex, float fillFactor, boolean verifyInput, long numElementsHint)
- throws HyracksDataException {
- this.lsmIndex = lsmIndex;
- this.component = lsmIndex.createBulkLoadTarget();
- this.componentBulkLoader = lsmIndex.createComponentBulkLoader(component, fillFactor, verifyInput,
- numElementsHint, false, true, true);
- }
-
- @Override
- public void add(ITupleReference tuple) throws HyracksDataException {
- componentBulkLoader.add(tuple);
- }
-
- @Override
- public void end() throws HyracksDataException {
- componentBulkLoader.end();
- if (component.getComponentSize() > 0) {
- //TODO(amoudi): Ensure Bulk load follow the same lifecycle Other Operations (Flush, Merge, etc).
- //then after operation should be called from harness as well
- //https://issues.apache.org/jira/browse/ASTERIXDB-1764
- lsmIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, component);
- lsmIndex.getLsmHarness().addBulkLoadedComponent(component);
- }
- }
-
- @Override
- public void abort() throws HyracksDataException {
- componentBulkLoader.end();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
index 4d60ab4..a8e707f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java
@@ -18,51 +18,47 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent {
- private final BTree btree;
- private final BloomFilter bloomFilter;
+public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent implements ILSMDiskComponent {
+ protected final BTree btree;
- public LSMBTreeDiskComponent(BTree btree, BloomFilter bloomFilter, ILSMComponentFilter filter) {
- super((IMetadataPageManager) btree.getPageManager(), filter);
+ public LSMBTreeDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, ILSMComponentFilter filter) {
+ super(lsmIndex, getMetadataPageManager(btree), filter);
this.btree = btree;
- this.bloomFilter = bloomFilter;
}
@Override
- public void destroy() throws HyracksDataException {
- btree.deactivate();
- btree.destroy();
- if (bloomFilter != null) {
- bloomFilter.deactivate();
- bloomFilter.destroy();
- }
- }
-
- public BTree getBTree() {
+ public BTree getIndex() {
return btree;
}
- public BloomFilter getBloomFilter() {
- return bloomFilter;
+ @Override
+ public BTree getMetadataHolder() {
+ return btree;
}
@Override
public long getComponentSize() {
- return btree.getFileReference().getFile().length()
- + (bloomFilter == null ? 0 : bloomFilter.getFileReference().getFile().length());
+ return getComponentSize(btree);
}
@Override
public int getFileReferenceCount() {
- return btree.getBufferCache().getFileReferenceCount(btree.getFileId());
+ return getFileReferenceCount(btree);
+ }
+
+ @Override
+ public Set<String> getLSMComponentPhysicalFiles() {
+ return getFiles(btree);
}
@Override
@@ -70,13 +66,21 @@ public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent {
return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath();
}
- @Override
- public void markAsValid(boolean persist) throws HyracksDataException {
- // The order of forcing the dirty page to be flushed is critical.
- // The bloom filter must be always done first.
- if (bloomFilter != null && persist) {
- ComponentUtils.markAsValid(btree.getBufferCache(), bloomFilter, persist);
- }
- ComponentUtils.markAsValid(btree, persist);
+ static IMetadataPageManager getMetadataPageManager(BTree btree) {
+ return (IMetadataPageManager) btree.getPageManager();
+ }
+
+ static long getComponentSize(BTree btree) {
+ return btree.getFileReference().getFile().length();
+ }
+
+ static int getFileReferenceCount(BTree btree) {
+ return btree.getBufferCache().getFileReferenceCount(btree.getFileId());
+ }
+
+ static Set<String> getFiles(BTree btree) {
+ Set<String> files = new HashSet<>();
+ files.add(btree.getFileReference().getFile().getAbsolutePath());
+ return files;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java
deleted file mode 100644
index 43373b5..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentBulkLoader.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.btree.impls;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentBulkLoader;
-import org.apache.hyracks.storage.common.IIndex;
-import org.apache.hyracks.storage.common.MultiComparator;
-
-public class LSMBTreeDiskComponentBulkLoader extends AbstractLSMDiskComponentBulkLoader {
-
- //with filter
- public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec,
- float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex,
- boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager, int[] indexFields,
- int[] filterFields, MultiComparator filterCmp) throws HyracksDataException {
- super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp);
- }
-
- //without filter
- public LSMBTreeDiskComponentBulkLoader(LSMBTreeDiskComponent component, BloomFilterSpecification bloomFilterSpec,
- float fillFactor, boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex,
- boolean cleanupEmptyComponent) throws HyracksDataException {
- super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- cleanupEmptyComponent, null, null, null, null);
- }
-
- @Override
- protected BloomFilter getBloomFilter(ILSMDiskComponent component) {
- return ((LSMBTreeDiskComponent) component).getBloomFilter();
- }
-
- @Override
- protected IIndex getIndex(ILSMDiskComponent component) {
- return ((LSMBTreeDiskComponent) component).getBTree();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
index 399904f..0e54f53 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentFactory.java
@@ -20,34 +20,27 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
public class LSMBTreeDiskComponentFactory implements ILSMDiskComponentFactory {
- private final TreeIndexFactory<BTree> btreeFactory;
- private final BloomFilterFactory bloomFilterFactory;
- private final IComponentFilterHelper filterHelper;
+ protected final TreeIndexFactory<BTree> btreeFactory;
+ protected final IComponentFilterHelper filterHelper;
- public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, BloomFilterFactory bloomFilterFactory,
- IComponentFilterHelper filterHelper) {
+ public LSMBTreeDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory, IComponentFilterHelper filterHelper) {
this.btreeFactory = btreeFactory;
- this.bloomFilterFactory = bloomFilterFactory;
this.filterHelper = filterHelper;
}
@Override
- public LSMBTreeDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
- return new LSMBTreeDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
- bloomFilterFactory == null ? null
- : bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ public LSMBTreeDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr)
+ throws HyracksDataException {
+ return new LSMBTreeDiskComponent(lsmIndex, btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
filterHelper == null ? null : filterHelper.createFilter());
}
- public int[] getBloomFilterKeyFields() {
- return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields();
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
index ab9b5af..02e481c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponentScanCursor.java
@@ -69,7 +69,6 @@ public class LSMBTreeDiskComponentScanCursor extends LSMIndexSearchCursor {
operationalComponents = lsmInitialState.getOperationalComponents();
lsmHarness = lsmInitialState.getLSMHarness();
includeMutableComponent = false;
-
int numBTrees = operationalComponents.size();
rangeCursors = new IIndexCursor[numBTrees];
btreeAccessors = new BTreeAccessor[numBTrees];
@@ -77,10 +76,9 @@ public class LSMBTreeDiskComponentScanCursor extends LSMIndexSearchCursor {
ILSMComponent component = operationalComponents.get(i);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
- BTree btree = ((LSMBTreeDiskComponent) component).getBTree();
+ BTree btree = (BTree) component.getIndex();
- btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
btreeAccessors[i].search(rangeCursors[i], searchPred);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
index 32b93bb..b334d80 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeMemoryComponent.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
@@ -29,23 +28,13 @@ public class LSMBTreeMemoryComponent extends AbstractLSMMemoryComponent {
private final BTree btree;
- public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive,
- ILSMComponentFilter filter) {
+ public LSMBTreeMemoryComponent(BTree btree, IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
super(vbc, isActive, filter);
this.btree = btree;
}
- public BTree getBTree() {
- return btree;
- }
-
@Override
- public void reset() throws HyracksDataException {
- super.reset();
- btree.deactivate();
- btree.destroy();
- btree.create();
- btree.activate();
+ public BTree getIndex() {
+ return btree;
}
-
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
index 86502a7..08ba7af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeOpContext.java
@@ -66,29 +66,29 @@ public final class LSMBTreeOpContext extends AbstractLSMIndexOperationContext {
ILSMHarness lsmHarness, IBinaryComparatorFactory[] filterCmpFactories) {
super(btreeFields, filterFields, filterCmpFactories, searchCallback, modificationCallback);
LSMBTreeMemoryComponent c = (LSMBTreeMemoryComponent) mutableComponents.get(0);
- IBinaryComparatorFactory cmpFactories[] = c.getBTree().getComparatorFactories();
+ IBinaryComparatorFactory cmpFactories[] = c.getIndex().getComparatorFactories();
if (cmpFactories[0] != null) {
- this.cmp = MultiComparator.create(c.getBTree().getComparatorFactories());
+ this.cmp = MultiComparator.create(c.getIndex().getComparatorFactories());
} else {
this.cmp = null;
}
bloomFilterCmp = numBloomFilterKeyFields == 0 ? null
- : MultiComparator.create(c.getBTree().getComparatorFactories(), 0, numBloomFilterKeyFields);
+ : MultiComparator.create(c.getIndex().getComparatorFactories(), 0, numBloomFilterKeyFields);
mutableBTrees = new BTree[mutableComponents.size()];
mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
mutableBTreeOpCtxs = new BTreeOpContext[mutableComponents.size()];
for (int i = 0; i < mutableComponents.size(); i++) {
LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) mutableComponents.get(i);
- mutableBTrees[i] = mutableComponent.getBTree();
+ mutableBTrees[i] = mutableComponent.getIndex();
if (allFields != null) {
- mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableBTrees[i].createAccessor(modificationCallback,
+ mutableBTreeAccessors[i] = mutableBTrees[i].createAccessor(modificationCallback,
NoOpOperationCallback.INSTANCE, allFields);
} else {
- mutableBTreeAccessors[i] = (BTree.BTreeAccessor) mutableBTrees[i].createAccessor(modificationCallback,
- NoOpOperationCallback.INSTANCE);
+ mutableBTreeAccessors[i] =
+ mutableBTrees[i].createAccessor(modificationCallback, NoOpOperationCallback.INSTANCE);
}
mutableBTreeOpCtxs[i] = mutableBTreeAccessors[i].getOpContext();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
index 6126c53..c178b51 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreePointSearchCursor.java
@@ -178,24 +178,24 @@ public class LSMBTreePointSearchCursor implements ITreeIndexCursor {
// reset
rangeCursors[i].reset();
}
- btree = ((LSMBTreeMemoryComponent) component).getBTree();
+ btree = ((LSMBTreeMemoryComponent) component).getIndex();
} else {
if (rangeCursors[i] != null && rangeCursors[i].isBloomFilterAware()) {
// can re-use cursor
((BloomFilterAwareBTreePointSearchCursor) rangeCursors[i])
- .resetBloomFilter(((LSMBTreeDiskComponent) component).getBloomFilter());
+ .resetBloomFilter(((LSMBTreeWithBloomFilterDiskComponent) component).getBloomFilter());
rangeCursors[i].reset();
} else {
// create new cursor <should be relatively rare>
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getLeafFrameFactory().createFrame();
rangeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(leafFrame, false,
- ((LSMBTreeDiskComponent) component).getBloomFilter());
+ ((LSMBTreeWithBloomFilterDiskComponent) component).getBloomFilter());
}
- btree = ((LSMBTreeDiskComponent) component).getBTree();
+ btree = (BTree) component.getIndex();
}
if (btreeAccessors[i] == null) {
- btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i] =
+ btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
} else {
// re-use
btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
index 29d7c60..7f93b62 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeRangeSearchCursor.java
@@ -231,14 +231,14 @@ public class LSMBTreeRangeSearchCursor extends LSMIndexSearchCursor {
}
if (component.getType() == LSMComponentType.MEMORY) {
includeMutableComponent = true;
- btree = ((LSMBTreeMemoryComponent) component).getBTree();
+ btree = (BTree) component.getIndex();
} else {
- btree = ((LSMBTreeDiskComponent) component).getBTree();
+ btree = (BTree) component.getIndex();
}
if (btreeAccessors[i] == null) {
- btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i] =
+ btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
} else {
// re-use
btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java
new file mode 100644
index 0000000..fa17549
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponent.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.impls;
+
+import java.util.Set;
+
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBloomFilterDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public class LSMBTreeWithBloomFilterDiskComponent extends AbstractLSMWithBloomFilterDiskComponent {
+
+ private final BTree btree;
+ private final BloomFilter bloomFilter;
+
+ public LSMBTreeWithBloomFilterDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, BloomFilter bloomFilter,
+ ILSMComponentFilter filter) {
+ super(lsmIndex, LSMBTreeDiskComponent.getMetadataPageManager(btree), filter);
+ this.btree = btree;
+ this.bloomFilter = bloomFilter;
+ }
+
+ @Override
+ public BloomFilter getBloomFilter() {
+ return bloomFilter;
+ }
+
+ @Override
+ public IBufferCache getBloomFilterBufferCache() {
+ return getMetadataHolder().getBufferCache();
+ }
+
+ @Override
+ public long getComponentSize() {
+ return LSMBTreeDiskComponent.getComponentSize(btree) + getComponentSize(bloomFilter);
+ }
+
+ @Override
+ public Set<String> getLSMComponentPhysicalFiles() {
+ Set<String> files = LSMBTreeDiskComponent.getFiles(btree);
+ addFiles(files, bloomFilter);
+ return files;
+ }
+
+ static void addFiles(Set<String> files, BloomFilter bloomFilter) {
+ files.add(bloomFilter.getFileReference().getFile().getAbsolutePath());
+ }
+
+ @Override
+ public int getFileReferenceCount() {
+ return LSMBTreeDiskComponent.getFileReferenceCount(btree);
+ }
+
+ @Override
+ public BTree getMetadataHolder() {
+ return btree;
+ }
+
+ @Override
+ public BTree getIndex() {
+ return btree;
+ }
+
+ static long getComponentSize(BloomFilter bloomFilter) {
+ return bloomFilter.getFileReference().getFile().length();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java
new file mode 100644
index 0000000..09e83cc
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBloomFilterDiskComponentFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.btree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
+import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+
+public class LSMBTreeWithBloomFilterDiskComponentFactory implements ILSMDiskComponentFactory {
+ protected final TreeIndexFactory<BTree> btreeFactory;
+ protected final IComponentFilterHelper filterHelper;
+ protected final BloomFilterFactory bloomFilterFactory;
+
+ public LSMBTreeWithBloomFilterDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory,
+ BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper) {
+ this.btreeFactory = btreeFactory;
+ this.filterHelper = filterHelper;
+ this.bloomFilterFactory = bloomFilterFactory;
+ }
+
+ @Override
+ public LSMBTreeWithBloomFilterDiskComponent createComponent(AbstractLSMIndex lsmIndex,
+ LSMComponentFileReferences cfr) throws HyracksDataException {
+ return new LSMBTreeWithBloomFilterDiskComponent(lsmIndex,
+ btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ filterHelper == null ? null : filterHelper.createFilter());
+ }
+
+ public int[] getBloomFilterKeyFields() {
+ return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
index 03d28d0..5122e43 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyAbstractCursor.java
@@ -33,6 +33,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterAwareBTreePointSearchCursor;
import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
@@ -104,26 +105,26 @@ public abstract class LSMBTreeWithBuddyAbstractCursor implements ITreeIndexCurso
} else {
buddyBtreeCursors[i].reset();
}
- btree = ((LSMBTreeWithBuddyMemoryComponent) component).getBTree();
- buddyBtree = ((LSMBTreeWithBuddyMemoryComponent) component).getBuddyBTree();
+ btree = ((LSMBTreeWithBuddyMemoryComponent) component).getIndex();
+ buddyBtree = ((LSMBTreeWithBuddyMemoryComponent) component).getBuddyIndex();
} else {
if (buddyBtreeCursors[i] == null || !buddyBtreeCursors[i].isBloomFilterAware()) {
buddyBtreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor(
(IBTreeLeafFrame) lsmInitialState.getBuddyBTreeLeafFrameFactory().createFrame(), false,
- ((LSMBTreeWithBuddyDiskComponent) operationalComponents.get(i)).getBloomFilter());
+ ((AbstractLSMWithBuddyDiskComponent) operationalComponents.get(i)).getBloomFilter());
} else {
buddyBtreeCursors[i].reset();
}
- btree = ((LSMBTreeWithBuddyDiskComponent) component).getBTree();
- buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
+ btree = (BTree) component.getIndex();
+ buddyBtree = (BTree) ((AbstractLSMWithBuddyDiskComponent) component).getBuddyIndex();
}
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame();
if (btreeAccessors[i] == null) {
btreeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
- btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
- buddyBtreeAccessors[i] = (BTreeAccessor) buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE,
- NoOpOperationCallback.INSTANCE);
+ btreeAccessors[i] =
+ btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
+ buddyBtreeAccessors[i] =
+ buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
} else {
btreeCursors[i].reset();
btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
index 4aee950..1e9dade 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java
@@ -18,72 +18,76 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.impls;
+import java.util.Set;
+
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
-
-public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMDiskComponent {
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMWithBuddyDiskComponent {
private final BTree btree;
- private final BTree buddyBtree;
private final BloomFilter bloomFilter;
+ private final BTree buddyBtree;
- public LSMBTreeWithBuddyDiskComponent(BTree btree, BTree buddyBtree, BloomFilter bloomFilter,
- ILSMComponentFilter filter) {
- super((IMetadataPageManager) btree.getPageManager(), filter);
+ public LSMBTreeWithBuddyDiskComponent(AbstractLSMIndex lsmIndex, BTree btree, BTree buddyBtree,
+ BloomFilter bloomFilter, ILSMComponentFilter filter) {
+ super(lsmIndex, LSMBTreeDiskComponent.getMetadataPageManager(btree), filter);
this.btree = btree;
- this.buddyBtree = buddyBtree;
this.bloomFilter = bloomFilter;
+ this.buddyBtree = buddyBtree;
}
@Override
- public void destroy() throws HyracksDataException {
- btree.deactivate();
- btree.destroy();
- buddyBtree.deactivate();
- buddyBtree.destroy();
- bloomFilter.deactivate();
- bloomFilter.destroy();
+ public BTree getBuddyIndex() {
+ return buddyBtree;
}
- public BTree getBTree() {
- return btree;
+ @Override
+ public long getComponentSize() {
+ return LSMBTreeDiskComponent.getComponentSize(btree)
+ + LSMBTreeWithBloomFilterDiskComponent.getComponentSize(bloomFilter)
+ + buddyBtree.getFileReference().getFile().length();
}
- public BTree getBuddyBTree() {
- return buddyBtree;
+ @Override
+ public Set<String> getLSMComponentPhysicalFiles() {
+ Set<String> files = LSMBTreeDiskComponent.getFiles(btree);
+ LSMBTreeWithBloomFilterDiskComponent.addFiles(files, bloomFilter);
+ files.add(buddyBtree.getFileReference().getFile().getAbsolutePath());
+ return files;
}
- public BloomFilter getBloomFilter() {
- return bloomFilter;
+ @Override
+ public void validate() throws HyracksDataException {
+ throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
}
@Override
- public long getComponentSize() {
- long size = btree.getFileReference().getFile().length();
- size += buddyBtree.getFileReference().getFile().length();
- size += bloomFilter.getFileReference().getFile().length();
- return size;
+ public int getFileReferenceCount() {
+ return LSMBTreeDiskComponent.getFileReferenceCount(btree);
}
@Override
- public int getFileReferenceCount() {
- return btree.getBufferCache().getFileReferenceCount(btree.getFileId());
+ public BTree getMetadataHolder() {
+ return btree;
}
@Override
- public String toString() {
- return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath();
+ public BTree getIndex() {
+ return btree;
+ }
+
+ @Override
+ public BloomFilter getBloomFilter() {
+ return bloomFilter;
}
@Override
- public void markAsValid(boolean persist) throws HyracksDataException {
- ComponentUtils.markAsValid(btree.getBufferCache(), bloomFilter, persist);
- ComponentUtils.markAsValid(btree, persist);
- ComponentUtils.markAsValid(buddyBtree, persist);
+ public IBufferCache getBloomFilterBufferCache() {
+ return getMetadataHolder().getBufferCache();
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java
deleted file mode 100644
index b3e7c6c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentBulkLoader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.btree.impls;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
-import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponentWithBuddyBulkLoader;
-import org.apache.hyracks.storage.common.IIndex;
-import org.apache.hyracks.storage.common.MultiComparator;
-
-public class LSMBTreeWithBuddyDiskComponentBulkLoader extends AbstractLSMDiskComponentWithBuddyBulkLoader {
-
- //with filter
- public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component,
- BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean cleanupEmptyComponent, ILSMComponentFilterManager filterManager,
- int[] indexFields, int[] filterFields, MultiComparator filterCmp) throws HyracksDataException {
- super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- cleanupEmptyComponent, filterManager, indexFields, filterFields, filterCmp);
- }
-
- //without filter
- public LSMBTreeWithBuddyDiskComponentBulkLoader(LSMBTreeWithBuddyDiskComponent component,
- BloomFilterSpecification bloomFilterSpec, float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, boolean cleanupEmptyComponent) throws HyracksDataException {
- super(component, bloomFilterSpec, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- cleanupEmptyComponent, null, null, null, null);
- }
-
- @Override
- protected BloomFilter getBloomFilter(ILSMDiskComponent component) {
- return ((LSMBTreeWithBuddyDiskComponent) component).getBloomFilter();
- }
-
- @Override
- protected IIndex getIndex(ILSMDiskComponent component) {
- return ((LSMBTreeWithBuddyDiskComponent) component).getBTree();
- }
-
- @Override
- protected ITreeIndex getBuddyBTree(ILSMDiskComponent component) {
- return ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
index 4223954..bfa2314 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponentFactory.java
@@ -21,27 +21,39 @@ package org.apache.hyracks.storage.am.lsm.btree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.impls.BTree;
+import org.apache.hyracks.storage.am.lsm.common.api.IComponentFilterHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
public class LSMBTreeWithBuddyDiskComponentFactory implements ILSMDiskComponentFactory {
- private final TreeIndexFactory<BTree> btreeFactory;
- private final TreeIndexFactory<BTree> buddyBtreeFactory;
- private final BloomFilterFactory bloomFilterFactory;
+ protected final TreeIndexFactory<BTree> btreeFactory;
+ protected final IComponentFilterHelper filterHelper;
+ protected final BloomFilterFactory bloomFilterFactory;
+ protected final TreeIndexFactory<BTree> buddyBtreeFactory;
public LSMBTreeWithBuddyDiskComponentFactory(TreeIndexFactory<BTree> btreeFactory,
- TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory) {
+ TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory,
+ IComponentFilterHelper filterHelper) {
this.btreeFactory = btreeFactory;
- this.buddyBtreeFactory = buddyBtreeFactory;
+ this.filterHelper = filterHelper;
this.bloomFilterFactory = bloomFilterFactory;
+ this.buddyBtreeFactory = buddyBtreeFactory;
+ }
+
+ public int[] getBloomFilterKeyFields() {
+ return bloomFilterFactory == null ? null : bloomFilterFactory.getBloomFilterKeyFields();
}
@Override
- public LSMBTreeWithBuddyDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException {
- return new LSMBTreeWithBuddyDiskComponent(btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
+ public LSMBTreeWithBuddyDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr)
+ throws HyracksDataException {
+ return new LSMBTreeWithBuddyDiskComponent(lsmIndex,
+ btreeFactory.createIndexInstance(cfr.getInsertIndexFileReference()),
buddyBtreeFactory.createIndexInstance(cfr.getDeleteIndexFileReference()),
- bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()), null);
+ bloomFilterFactory.createBloomFiltertInstance(cfr.getBloomFilterFileReference()),
+ filterHelper == null ? null : filterHelper.createFilter());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
index f58c737..828b58e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyMemoryComponent.java
@@ -21,13 +21,13 @@ package org.apache.hyracks.storage.am.lsm.btree.impls;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.btree.impls.BTree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilter;
+import org.apache.hyracks.storage.am.lsm.common.api.AbstractLSMWithBuddyMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
/*
* This class is also not needed at the moment but is implemented anyway
*/
-public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMMemoryComponent {
+public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMWithBuddyMemoryComponent {
private final BTree btree;
private final BTree buddyBtree;
@@ -39,25 +39,23 @@ public class LSMBTreeWithBuddyMemoryComponent extends AbstractLSMMemoryComponent
this.buddyBtree = buddyBtree;
}
- public BTree getBTree() {
+ @Override
+ public BTree getIndex() {
return btree;
}
- public BTree getBuddyBTree() {
+ @Override
+ public BTree getBuddyIndex() {
return buddyBtree;
}
@Override
- public void reset() throws HyracksDataException {
- super.reset();
- btree.deactivate();
- btree.destroy();
- btree.create();
- btree.activate();
- buddyBtree.deactivate();
- buddyBtree.destroy();
- buddyBtree.create();
- buddyBtree.activate();
+ public void validate() throws HyracksDataException {
+ throw new UnsupportedOperationException("Validation not implemented for LSM B-Trees with Buddy B-Tree.");
}
+ @Override
+ public long getSize() {
+ return 0L;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
index 79d6b0f..061cad5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBuddyBTreeMergeCursor.java
@@ -62,7 +62,7 @@ public class LSMBuddyBTreeMergeCursor extends LSMIndexSearchCursor {
ILSMComponent component = operationalComponents.get(i);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getBuddyBTreeLeafFrameFactory().createFrame();
rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
- BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
+ BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyIndex();
btreeAccessors[i] =
buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
btreeAccessors[i].search(rangeCursors[i], btreePredicate);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 4aafbe9..2c4e6b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -37,10 +37,14 @@ import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree;
import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBloomFilterDiskComponentFactory;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory;
import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
@@ -89,9 +93,6 @@ public class LSMBTreeUtil {
new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
- BloomFilterFactory bloomFilterFactory =
- needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null;
-
ComponentFilterHelper filterHelper = null;
LSMComponentFilterFrameFactory filterFrameFactory = null;
LSMComponentFilterManager filterManager = null;
@@ -106,8 +107,21 @@ public class LSMBTreeUtil {
ILSMIndexFileManager fileNameManager =
new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck);
+ ILSMDiskComponentFactory componentFactory;
+ ILSMDiskComponentFactory bulkLoadComponentFactory;
+ if (needKeyDupCheck) {
+ BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+ componentFactory =
+ new LSMBTreeWithBloomFilterDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, filterHelper);
+ bulkLoadComponentFactory = new LSMBTreeWithBloomFilterDiskComponentFactory(bulkLoadBTreeFactory,
+ bloomFilterFactory, filterHelper);
+ } else {
+ componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, filterHelper);
+ bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, filterHelper);
+ }
+
return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
- deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
+ deleteLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory,
filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields,
filterFields, durable, updateAware, tracer);
@@ -149,11 +163,19 @@ public class LSMBTreeUtil {
transactionLeafFrameFactory, cmpFactories, typeTraits.length);
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true);
+
+ ILSMDiskComponentFactory componentFactory =
+ new LSMBTreeWithBloomFilterDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
+ ILSMDiskComponentFactory bulkLoadComponentFactory =
+ new LSMBTreeWithBloomFilterDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
+ ILSMDiskComponentFactory transactionComponentFactory =
+ new LSMBTreeWithBloomFilterDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
+
// the disk only index uses an empty ArrayList for virtual buffer caches
return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
- fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback,
- transactionBTreeFactory, durable);
+ diskBufferCache, fileNameManager, componentFactory, bulkLoadComponentFactory,
+ transactionComponentFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker,
+ ioScheduler, ioOpCallback, durable);
}
public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file,
@@ -200,12 +222,15 @@ public class LSMBTreeUtil {
ILSMIndexFileManager fileNameManager =
new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory);
+ ILSMDiskComponentFactory componentFactory = new LSMBTreeWithBuddyDiskComponentFactory(diskBTreeFactory,
+ diskBuddyBTreeFactory, bloomFilterFactory, null);
+ ILSMDiskComponentFactory bulkLoadComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory(
+ bulkLoadBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, null);
+
// the disk only index uses an empty ArrayList for virtual buffer caches
- ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory,
- insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager,
- bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory,
- bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories,
- buddyBtreeCmpFactories, buddyBTreeFields, durable);
- return lsmTree;
+ return new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory,
+ buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, componentFactory,
+ bulkLoadComponentFactory, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+ ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
new file mode 100644
index 0000000..107190d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBloomFilterDiskComponent.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.BloomFilterBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public abstract class AbstractLSMWithBloomFilterDiskComponent extends AbstractLSMDiskComponent {
+ public AbstractLSMWithBloomFilterDiskComponent(AbstractLSMIndex lsmIndex, IMetadataPageManager mdPageManager,
+ ILSMComponentFilter filter) {
+ super(lsmIndex, mdPageManager, filter);
+ }
+
+ public abstract BloomFilter getBloomFilter();
+
+ public abstract IBufferCache getBloomFilterBufferCache();
+
+ @Override
+ public void markAsValid(boolean persist) throws HyracksDataException {
+ // The order of forcing the dirty page to be flushed is critical. The
+ // bloom filter must be always done first.
+ ComponentUtils.markAsValid(getBloomFilterBufferCache(), getBloomFilter(), persist);
+ super.markAsValid(persist);
+ }
+
+ @Override
+ public void activate(boolean createNewComponent) throws HyracksDataException {
+ super.activate(createNewComponent);
+ if (createNewComponent) {
+ getBloomFilter().create();
+ }
+ getBloomFilter().activate();
+ }
+
+ @Override
+ public void deactivateAndDestroy() throws HyracksDataException {
+ super.deactivateAndDestroy();
+ getBloomFilter().deactivate();
+ getBloomFilter().destroy();
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ super.destroy();
+ getBloomFilter().destroy();
+ }
+
+ @Override
+ public void deactivate() throws HyracksDataException {
+ super.deactivate();
+ getBloomFilter().deactivate();
+ }
+
+ @Override
+ public void deactivateAndPurge() throws HyracksDataException {
+ super.deactivateAndPurge();
+ getBloomFilter().deactivate();
+ getBloomFilter().purge();
+ }
+
+ public IChainedComponentBulkLoader createBloomFilterBulkLoader(long numElementsHint) throws HyracksDataException {
+ BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(
+ BloomCalculations.maxBucketsPerElement(numElementsHint), getLsmIndex().bloomFilterFalsePositiveRate());
+ return new BloomFilterBulkLoader(getBloomFilter().createBuilder(numElementsHint, bloomFilterSpec.getNumHashes(),
+ bloomFilterSpec.getNumBucketsPerElements()));
+ }
+
+ @Override
+ public ChainedLSMDiskComponentBulkLoader createBulkLoader(float fillFactor, boolean verifyInput,
+ long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter, boolean cleanupEmptyComponent)
+ throws HyracksDataException {
+ ChainedLSMDiskComponentBulkLoader chainedBulkLoader = super.createBulkLoader(fillFactor, verifyInput,
+ numElementsHint, checkIfEmptyIndex, withFilter, cleanupEmptyComponent);
+ if (numElementsHint > 0) {
+ chainedBulkLoader.addBulkLoader(createBloomFilterBulkLoader(numElementsHint));
+ }
+ return chainedBulkLoader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java
new file mode 100644
index 0000000..c2f52e0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyDiskComponent.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.impls.IndexWithBuddyBulkLoader;
+import org.apache.hyracks.storage.am.lsm.common.util.ComponentUtils;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+
+public abstract class AbstractLSMWithBuddyDiskComponent extends AbstractLSMWithBloomFilterDiskComponent {
+
+ public AbstractLSMWithBuddyDiskComponent(AbstractLSMIndex lsmIndex, IMetadataPageManager mdPageManager,
+ ILSMComponentFilter filter) {
+ super(lsmIndex, mdPageManager, filter);
+ }
+
+ public abstract AbstractTreeIndex getBuddyIndex();
+
+ @Override
+ public void markAsValid(boolean persist) throws HyracksDataException {
+ super.markAsValid(persist);
+ ComponentUtils.markAsValid(getBuddyIndex(), persist);
+ }
+
+ @Override
+ public void activate(boolean createNewComponent) throws HyracksDataException {
+ super.activate(createNewComponent);
+ if (createNewComponent) {
+ getBuddyIndex().create();
+ }
+ getBuddyIndex().activate();
+ }
+
+ @Override
+ public void deactivateAndDestroy() throws HyracksDataException {
+ super.deactivateAndDestroy();
+ getBuddyIndex().deactivate();
+ getBuddyIndex().destroy();
+ }
+
+ @Override
+ public void destroy() throws HyracksDataException {
+ super.destroy();
+ getBuddyIndex().destroy();
+ }
+
+ @Override
+ public void deactivate() throws HyracksDataException {
+ super.deactivate();
+ getBuddyIndex().deactivate();
+ }
+
+ @Override
+ public void deactivateAndPurge() throws HyracksDataException {
+ super.deactivateAndPurge();
+ getBuddyIndex().deactivate();
+ getBuddyIndex().purge();
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ super.validate();
+ getBuddyIndex().validate();
+ }
+
+ @Override
+ public IChainedComponentBulkLoader createIndexBulkLoader(float fillFactor, boolean verifyInput,
+ long numElementsHint, boolean checkIfEmptyIndex) throws HyracksDataException {
+ IIndexBulkLoader indexBulkLoader =
+ getIndex().createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
+ IIndexBulkLoader buddyBulkLoader =
+ getBuddyIndex().createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
+ return new IndexWithBuddyBulkLoader(indexBulkLoader, buddyBulkLoader);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java
new file mode 100644
index 0000000..b445142
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/AbstractLSMWithBuddyMemoryComponent.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMMemoryComponent;
+
+public abstract class AbstractLSMWithBuddyMemoryComponent extends AbstractLSMMemoryComponent {
+
+ public AbstractLSMWithBuddyMemoryComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter filter) {
+ super(vbc, isActive, filter);
+ }
+
+ public abstract AbstractTreeIndex getBuddyIndex();
+
+ @Override
+ public void doReset() throws HyracksDataException {
+ super.doReset();
+ getBuddyIndex().deactivate();
+ getBuddyIndex().destroy();
+ getBuddyIndex().create();
+ getBuddyIndex().activate();
+ }
+
+ @Override
+ public void doAllocate() throws HyracksDataException {
+ super.doAllocate();
+ getBuddyIndex().create();
+ getBuddyIndex().activate();
+ }
+
+ @Override
+ public void doDeallocate() throws HyracksDataException {
+ super.doDeallocate();
+ getBuddyIndex().deactivate();
+ getBuddyIndex().destroy();
+ }
+
+ @Override
+ public void validate() throws HyracksDataException {
+ super.validate();
+ getBuddyIndex().validate();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
index a446f4e..a60f544 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponent.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.IIndex;
/**
* An LSM index component. can be an in memory or on disk. Can be readable or unreadable, writable or unwritable
@@ -121,4 +122,9 @@ public interface ILSMComponent {
* @return the component filter
*/
ILSMComponentFilter getLSMComponentFilter();
+
+ /**
+ * @return index data structure that is the stored in the component
+ */
+ IIndex getIndex();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
index b101315..43c5482 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponent.java
@@ -18,8 +18,14 @@
*/
package org.apache.hyracks.storage.am.lsm.common.api;
+import java.util.Set;
+
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.ChainedLSMDiskComponentBulkLoader;
import org.apache.hyracks.storage.am.lsm.common.impls.DiskComponentMetadata;
+import org.apache.hyracks.storage.am.lsm.common.impls.IChainedComponentBulkLoader;
public interface ILSMDiskComponent extends ILSMComponent {
@@ -42,13 +48,6 @@ public interface ILSMDiskComponent extends ILSMComponent {
int getFileReferenceCount();
/**
- * Delete the component from disk
- *
- * @throws HyracksDataException
- */
- void destroy() throws HyracksDataException;
-
- /**
* Return the component Id of this disk component from its metadata
*
* @return
@@ -57,6 +56,21 @@ public interface ILSMDiskComponent extends ILSMComponent {
ILSMDiskComponentId getComponentId() throws HyracksDataException;
/**
+ * @return LsmIndex of the component
+ */
+ AbstractLSMIndex getLsmIndex();
+
+ /**
+ * @return the TreeIndex which holds metadata for the disk component
+ */
+ ITreeIndex getMetadataHolder();
+
+ /**
+ * @return a set of files describing the contents of the disk component
+ */
+ Set<String> getLSMComponentPhysicalFiles();
+
+ /**
* Mark the component as valid
*
* @param persist
@@ -65,4 +79,86 @@ public interface ILSMDiskComponent extends ILSMComponent {
*/
void markAsValid(boolean persist) throws HyracksDataException;
+ /**
+ * Activates the component
+ *
+ * @param create
+ * whether a new component should be created
+ * @throws HyracksDataException
+ */
+ void activate(boolean create) throws HyracksDataException;
+
+ /**
+ * Deactivate and destroy the component (Deletes it from disk)
+ *
+ * @throws HyracksDataException
+ */
+ void deactivateAndDestroy() throws HyracksDataException;
+
+ /**
+ * Destroy the component (Deletes it from disk)
+ *
+ * @throws HyracksDataException
+ */
+ void destroy() throws HyracksDataException;
+
+ /**
+ * Deactivate the component
+ * The pages are still in the buffer cache
+ *
+ * @throws HyracksDataException
+ */
+ void deactivate() throws HyracksDataException;
+
+ /**
+ * Deactivate the component and purge it out of the buffer cache
+ *
+ * @throws HyracksDataException
+ */
+ void deactivateAndPurge() throws HyracksDataException;
+
+ /**
+ * Test method. validates the content of the component
+ * TODO: Remove this method from the interface
+ *
+ * @throws HyracksDataException
+ */
+ void validate() throws HyracksDataException;
+
+ /**
+ * Creates a chained bulkloader which populates component's LSM filter
+ *
+ * @return
+ * @throws HyracksDataException
+ */
+ IChainedComponentBulkLoader createFilterBulkLoader() throws HyracksDataException;
+
+ /**
+ * Creates a chained bulkloader which populates component's index
+ *
+ * @param fillFactor
+ * @param verifyInput
+ * @param numElementsHint
+ * @param checkIfEmptyIndex
+ * @return
+ * @throws HyracksDataException
+ */
+ IChainedComponentBulkLoader createIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex) throws HyracksDataException;
+
+ /**
+ * Creates a bulkloader pipeline which includes all chained operations, bulkloading individual elements of the
+ * component: indexes, LSM filters, Bloom filters, buddy indexes, etc.
+ *
+ * @param fillFactor
+ * @param verifyInput
+ * @param numElementsHint
+ * @param checkIfEmptyIndex
+ * @param withFilter
+ * @param cleanupEmptyComponent
+ * @return
+ * @throws HyracksDataException
+ */
+ ChainedLSMDiskComponentBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex, boolean withFilter, boolean cleanupEmptyComponent) throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
index f1d1ce1..13ce971 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentBulkLoader.java
@@ -34,4 +34,11 @@ public interface ILSMDiskComponentBulkLoader extends IIndexBulkLoader {
*/
void delete(ITupleReference tuple) throws HyracksDataException;
+ /**
+ * Releases all resources allocated during the bulkloading process
+ *
+ * @throws HyracksDataException
+ */
+ void cleanupArtifacts() throws HyracksDataException;
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
index 9daf30b..a904670 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMDiskComponentFactory.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.storage.am.lsm.common.api;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@FunctionalInterface
@@ -32,5 +33,6 @@ public interface ILSMDiskComponentFactory {
* @return a disk component
* @throws HyracksDataException
*/
- ILSMDiskComponent createComponent(LSMComponentFileReferences cfr) throws HyracksDataException;
+ ILSMDiskComponent createComponent(AbstractLSMIndex lsmIndex, LSMComponentFileReferences cfr)
+ throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/77f89525/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index addeb27..e4a9436 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -134,23 +134,6 @@ public interface ILSMIndex extends IIndex {
void updateFilter(ILSMIndexOperationContext ictx, ITupleReference tuple) throws HyracksDataException;
/**
- * Create a component bulk loader for the given component
- *
- * @param component
- * @param fillFactor
- * @param verifyInput
- * @param numElementsHint
- * @param checkIfEmptyIndex
- * @param withFilter
- * @param cleanupEmptyComponent
- * @return
- * @throws HyracksDataException
- */
- ILSMDiskComponentBulkLoader createComponentBulkLoader(ILSMDiskComponent component, float fillFactor,
- boolean verifyInput, long numElementsHint, boolean checkIfEmptyIndex, boolean withFilter,
- boolean cleanupEmptyComponent) throws HyracksDataException;
-
- /**
* Creates a disk component for the bulk load operation
*
* @return