You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2017/05/25 06:04:09 UTC
[2/6] asterixdb git commit: Fix upsert deadlock and upsert with
filtered primary only
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
index 76cf339..73be537 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTreeOpContext.java
@@ -18,31 +18,17 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.LinkedList;
-import java.util.List;
-
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
- private IndexOperation op;
private MultiComparator bTreeCmp;
private MultiComparator rTreeCmp;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private final ISearchOperationCallback searchCallback;
private final int targetIndexVersion;
- private ISearchPredicate searchPredicate;
private LSMRTreeCursorInitialState initialState;
public ExternalRTreeOpContext(IBinaryComparatorFactory[] rtreeCmpFactories,
@@ -50,10 +36,7 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
int targetIndexVersion, ILSMHarness lsmHarness, int[] comparatorFields,
IBinaryComparatorFactory[] linearizerArray, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory) {
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.searchCallback = searchCallback;
+ super(null, null, null, searchCallback, null);
this.targetIndexVersion = targetIndexVersion;
this.bTreeCmp = MultiComparator.create(btreeCmpFactories);
this.rTreeCmp = MultiComparator.create(rtreeCmpFactories);
@@ -63,29 +46,10 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
// Do nothing. this should never be called for disk only indexes
}
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- componentsToBeReplicated.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
public MultiComparator getBTreeMultiComparator() {
return bTreeCmp;
}
@@ -94,46 +58,10 @@ public class ExternalRTreeOpContext extends AbstractLSMIndexOperationContext {
return rTreeCmp;
}
- @Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- // This should never be needed for disk only indexes
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return null;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
public int getTargetIndexVersion() {
return targetIndexVersion;
}
- @Override
- public void setSearchPredicate(ISearchPredicate searchPredicate) {
- this.searchPredicate = searchPredicate;
- }
-
- @Override
- public ISearchPredicate getSearchPredicate() {
- return searchPredicate;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
public LSMRTreeCursorInitialState getInitialState() {
return initialState;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 8882639..640b311 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -45,25 +45,23 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.DualTupleReference;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
@@ -73,11 +71,10 @@ import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTree extends AbstractLSMRTree {
- protected int[] buddyBTreeFields;
+ protected final int[] buddyBTreeFields;
public LSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -124,92 +121,42 @@ public class LSMRTree extends AbstractLSMRTree {
this.buddyBTreeFields = buddyBTreeFields;
}
- /**
- * Opens LSMRTree, cleaning up invalid files from base dir, and registering
- * all valid files as on-disk RTrees and BTrees.
- *
- * @throws HyracksDataException
- */
- @Override
- public synchronized void activate() throws HyracksDataException {
- super.activate();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- List<LSMComponentFileReferences> validFileReferences;
- validFileReferences = fileManager.cleanupAndGetValidFiles();
- immutableComponents.clear();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeDiskComponent component;
- component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- lsmComonentFileReference.getDeleteIndexFileReference(),
- lsmComonentFileReference.getBloomFilterFileReference(), false);
- immutableComponents.add(component);
- }
- isActivated = true;
- }
-
- @Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- super.deactivate(flushOnExit);
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- RTree rtree = component.getRTree();
- BTree btree = component.getBTree();
- BloomFilter bloomFilter = component.getBloomFilter();
- rtree.deactivateCloseHandle();
- btree.deactivateCloseHandle();
- bloomFilter.deactivate();
- }
- isActivated = false;
- }
-
@Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences lsmComonentFileReferences)
+ throws HyracksDataException {
+ return createDiskComponent(componentFactory, lsmComonentFileReferences.getInsertIndexFileReference(),
+ lsmComonentFileReferences.getDeleteIndexFileReference(),
+ lsmComonentFileReferences.getBloomFilterFileReference(), false);
}
@Override
- public synchronized void destroy() throws HyracksDataException {
- super.destroy();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getRTree().destroy();
- }
- fileManager.deleteDirs();
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ RTree rtree = component.getRTree();
+ BTree btree = component.getBTree();
+ BloomFilter bloomFilter = component.getBloomFilter();
+ rtree.deactivateCloseHandle();
+ btree.deactivateCloseHandle();
+ bloomFilter.deactivate();
}
@Override
- public synchronized void clear() throws HyracksDataException {
- super.clear();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
- component.getBTree().deactivate();
- component.getBloomFilter().deactivate();
- component.getRTree().deactivate();
- component.getBTree().destroy();
- component.getBloomFilter().destroy();
- component.getRTree().destroy();
- }
- immutableComponents.clear();
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
+ component.getRTree().destroy();
}
@Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
- ILSMIndexOperationContext rctx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.FLUSH);
- rctx.getComponentHolder().addAll(ctx.getComponentHolder());
- LSMRTreeAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent,
- componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
+ component.getBTree().deactivate();
+ component.getBloomFilter().deactivate();
+ component.getRTree().deactivate();
+ component.getBTree().destroy();
+ component.getBloomFilter().destroy();
+ component.getRTree().destroy();
}
@Override
@@ -226,8 +173,8 @@ public class LSMRTree extends AbstractLSMRTree {
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(),
- flushOp.getBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(),
+ flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
RTree diskRTree = component.getRTree();
IIndexBulkLoader rTreeBulkloader;
ITreeIndexCursor cursor;
@@ -314,8 +261,8 @@ public class LSMRTree extends AbstractLSMRTree {
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
// Note. If we change the filter to write to metadata object, we don't need the if block above
flushingComponent.getMetadata().copy(component.getMetadata());
@@ -324,31 +271,16 @@ public class LSMRTree extends AbstractLSMRTree {
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMIndexOperationContext rctx =
- createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
- LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
- mergeOp.getBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+ LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+ mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
// In case we must keep the deleted-keys BTrees, then they must be merged *before* merging the r-trees so that
// lsmHarness.endSearch() is called once when the r-trees have been merged.
@@ -392,8 +324,8 @@ public class LSMRTree extends AbstractLSMRTree {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
+ getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
}
btreeBulkLoader.end();
@@ -415,47 +347,8 @@ public class LSMRTree extends AbstractLSMRTree {
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
- }
-
- public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
- private final DualTupleReference dualTuple;
-
- public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- dualTuple = new DualTupleReference(buddyBTreeFields);
- }
-
- @Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMRTreeSearchCursor(ctx, buddyBTreeFields);
- }
-
- @Override
- public void delete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- lsmHarness.modify(ctx, false, dualTuple);
- }
-
- @Override
- public boolean tryDelete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- return lsmHarness.modify(ctx, true, dualTuple);
- }
-
- @Override
- public void forceDelete(ITupleReference tuple) throws HyracksDataException {
- ctx.setOperation(IndexOperation.DELETE);
- dualTuple.reset(tuple);
- lsmHarness.forceModify(ctx, dualTuple);
- }
-
- public MultiComparator getMultiComparator() {
- LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.getCurrentRTreeOpContext().getCmp();
- }
+ return new LSMRTreeAccessor(getLsmHarness(), createOpContext(modificationCallback, searchCallback),
+ buddyBTreeFields);
}
protected ILSMDiskComponent createBulkLoadTarget() throws HyracksDataException {
@@ -465,9 +358,9 @@ public class LSMRTree extends AbstractLSMRTree {
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMRTreeBulkLoader(this, fillLevel, verifyInput, numElementsHint);
}
// This function is modified for R-Trees without antimatter tuples to allow buddy B-Tree to have only primary keys
@@ -503,114 +396,7 @@ public class LSMRTree extends AbstractLSMRTree {
}
}
}
- if (ctx.getFilterTuple() != null) {
- ctx.getFilterTuple().reset(tuple);
- memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
- ctx.getFilterCmp());
- }
- }
-
- public class LSMRTreeBulkLoader implements IIndexBulkLoader {
- private final ILSMDiskComponent component;
- private final IIndexBulkLoader bulkLoader;
- private final IIndexBulkLoader buddyBTreeBulkloader;
- private boolean cleanedUpArtifacts = false;
- private boolean isEmptyComponent = true;
- public final PermutingTupleReference indexTuple;
- public final PermutingTupleReference filterTuple;
- public final MultiComparator filterCmp;
-
- public LSMRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
- // Note that by using a flush target file name, we state that the
- // new bulk loaded tree is "newer" than any other merged tree.
- component = createBulkLoadTarget();
- bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
- numElementsHint, false);
- buddyBTreeBulkloader = ((LSMRTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
- verifyInput, numElementsHint, false);
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
- filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
- }
-
- @Override
- public void add(ITupleReference tuple) throws HyracksDataException {
- try {
- ITupleReference t;
- if (indexTuple != null) {
- indexTuple.reset(tuple);
- t = indexTuple;
- } else {
- t = tuple;
- }
-
- bulkLoader.add(t);
-
- if (filterTuple != null) {
- filterTuple.reset(tuple);
- component.getLSMComponentFilter().update(filterTuple, filterCmp);
- }
- } catch (Exception e) {
- cleanupArtifacts();
- throw e;
- }
- if (isEmptyComponent) {
- isEmptyComponent = false;
- }
- }
-
- @Override
- public void end() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
-
- if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
- ((LSMRTreeDiskComponent) component).getRTree());
- }
-
- bulkLoader.end();
- buddyBTreeBulkloader.end();
-
- if (isEmptyComponent) {
- cleanupArtifacts();
- } else {
- ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
- }
- }
- }
-
- @Override
- public void abort() throws HyracksDataException {
- if (bulkLoader != null) {
- bulkLoader.abort();
- }
- if (buddyBTreeBulkloader != null) {
- buddyBTreeBulkloader.abort();
- }
- }
-
- protected void cleanupArtifacts() throws HyracksDataException {
- if (!cleanedUpArtifacts) {
- cleanedUpArtifacts = true;
- ((LSMRTreeDiskComponent) component).getRTree().deactivate();
- ((LSMRTreeDiskComponent) component).getRTree().destroy();
- ((LSMRTreeDiskComponent) component).getBTree().deactivate();
- ((LSMRTreeDiskComponent) component).getBTree().destroy();
- ((LSMRTreeDiskComponent) component).getBloomFilter().deactivate();
- ((LSMRTreeDiskComponent) component).getBloomFilter().destroy();
- }
- }
+ updateFilter(ctx, tuple);
}
@Override
@@ -631,4 +417,24 @@ public class LSMRTree extends AbstractLSMRTree {
return files;
}
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ LSMRTreeAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields);
+ return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+ callback, fileManager.getBaseDir());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ ITreeIndexCursor cursor = new LSMRTreeSortedCursor(opCtx, linearizer, buddyBTreeFields);
+ ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields);
+ return new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
+ mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(),
+ mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
index b348020..edb7481 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAbstractCursor.java
@@ -44,7 +44,6 @@ import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
@@ -154,12 +153,6 @@ public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
}
@Override
- public ICachedPage getPage() {
- // do nothing
- return null;
- }
-
- @Override
public void close() throws HyracksDataException {
if (!open) {
return;
@@ -198,12 +191,8 @@ public abstract class LSMRTreeAbstractCursor implements ITreeIndexCursor {
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java
new file mode 100644
index 0000000..08748e2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeAccessor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.rtree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.common.tuples.DualTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+
+public class LSMRTreeAccessor extends LSMTreeIndexAccessor {
+ private final DualTupleReference dualTuple;
+
+ public LSMRTreeAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx, int[] buddyBTreeFields) {
+ super(lsmHarness, ctx, aCtx -> new LSMRTreeSearchCursor(aCtx, buddyBTreeFields));
+ dualTuple = new DualTupleReference(buddyBTreeFields);
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ lsmHarness.modify(getCtx(), false, dualTuple);
+ }
+
+ @Override
+ public boolean tryDelete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ return lsmHarness.modify(getCtx(), true, dualTuple);
+ }
+
+ @Override
+ public void forceDelete(ITupleReference tuple) throws HyracksDataException {
+ getCtx().setOperation(IndexOperation.DELETE);
+ dualTuple.reset(tuple);
+ lsmHarness.forceModify(getCtx(), dualTuple);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java
new file mode 100644
index 0000000..edc3e7d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeBulkLoader.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.rtree.impls;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.MultiComparator;
+
+public class LSMRTreeBulkLoader implements IIndexBulkLoader {
+ private final ILSMDiskComponent component;
+ private final IIndexBulkLoader bulkLoader;
+ private final IIndexBulkLoader buddyBTreeBulkloader;
+ private boolean cleanedUpArtifacts = false;
+ private boolean isEmptyComponent = true;
+ public final PermutingTupleReference indexTuple;
+ public final PermutingTupleReference filterTuple;
+ public final MultiComparator filterCmp;
+ private final LSMRTree lsmIndex;
+
+ public LSMRTreeBulkLoader(LSMRTree lsmIndex, float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ this.lsmIndex = lsmIndex;
+ // Note that by using a flush target file name, we state that the
+ // new bulk loaded tree is "newer" than any other merged tree.
+ component = lsmIndex.createBulkLoadTarget();
+ bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
+ numElementsHint, false);
+ buddyBTreeBulkloader = ((LSMRTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor, verifyInput,
+ numElementsHint, false);
+ if (lsmIndex.getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(lsmIndex.getTreeFields());
+ filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
+ filterTuple = new PermutingTupleReference(lsmIndex.getFilterFields());
+ } else {
+ indexTuple = null;
+ filterCmp = null;
+ filterTuple = null;
+ }
+ }
+
+ @Override
+ public void add(ITupleReference tuple) throws HyracksDataException {
+ try {
+ ITupleReference t;
+ if (indexTuple != null) {
+ indexTuple.reset(tuple);
+ t = indexTuple;
+ } else {
+ t = tuple;
+ }
+
+ bulkLoader.add(t);
+
+ if (filterTuple != null) {
+ filterTuple.reset(tuple);
+ component.getLSMComponentFilter().update(filterTuple, filterCmp);
+ }
+ } catch (Exception e) {
+ cleanupArtifacts();
+ throw e;
+ }
+ if (isEmptyComponent) {
+ isEmptyComponent = false;
+ }
+ }
+
+ @Override
+ public void end() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+
+ if (component.getLSMComponentFilter() != null) {
+ lsmIndex.getFilterManager().writeFilter(component.getLSMComponentFilter(),
+ ((LSMRTreeDiskComponent) component).getRTree());
+ }
+
+ bulkLoader.end();
+ buddyBTreeBulkloader.end();
+
+ if (isEmptyComponent) {
+ cleanupArtifacts();
+ } else {
+ lsmIndex.getIOOperationCallback().afterOperation(LSMOperationType.FLUSH, null, component);
+ lsmIndex.getLsmHarness().addBulkLoadedComponent(component);
+ }
+ }
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ if (bulkLoader != null) {
+ bulkLoader.abort();
+ }
+ if (buddyBTreeBulkloader != null) {
+ buddyBTreeBulkloader.abort();
+ }
+ }
+
+ protected void cleanupArtifacts() throws HyracksDataException {
+ if (!cleanedUpArtifacts) {
+ cleanedUpArtifacts = true;
+ ((LSMRTreeDiskComponent) component).getRTree().deactivate();
+ ((LSMRTreeDiskComponent) component).getRTree().destroy();
+ ((LSMRTreeDiskComponent) component).getBTree().deactivate();
+ ((LSMRTreeDiskComponent) component).getBTree().destroy();
+ ((LSMRTreeDiskComponent) component).getBloomFilter().deactivate();
+ ((LSMRTreeDiskComponent) component).getBloomFilter().destroy();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
index 618b3a7..f3e45ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFlushOperation.java
@@ -18,95 +18,30 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
-public class LSMRTreeFlushOperation implements ILSMIOOperation, Comparable<LSMRTreeFlushOperation> {
+public class LSMRTreeFlushOperation extends FlushOperation {
- private final ILSMIndexAccessor accessor;
- private final ILSMComponent flushingComponent;
- private final FileReference rtreeFlushTarget;
private final FileReference btreeFlushTarget;
private final FileReference bloomFilterFlushTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
- FileReference rtreeFlushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget,
+ public LSMRTreeFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent,
+ FileReference flushTarget, FileReference btreeFlushTarget, FileReference bloomFilterFlushTarget,
ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.flushingComponent = flushingComponent;
- this.rtreeFlushTarget = rtreeFlushTarget;
+ super(accessor, flushingComponent, flushTarget, callback, indexIdentifier);
this.btreeFlushTarget = btreeFlushTarget;
this.bloomFilterFlushTarget = bloomFilterFlushTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
- }
-
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- return Collections.emptySet();
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(rtreeFlushTarget.getDeviceHandle());
- if (btreeFlushTarget != null) {
- devs.add(btreeFlushTarget.getDeviceHandle());
- devs.add(bloomFilterFlushTarget.getDeviceHandle());
- }
- return devs;
}
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.flush(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getRTreeFlushTarget() {
- return rtreeFlushTarget;
- }
-
- public FileReference getBTreeFlushTarget() {
+ public FileReference getBTreeTarget() {
return btreeFlushTarget;
}
- public FileReference getBloomFilterFlushTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterFlushTarget;
}
-
- public ILSMComponent getFlushingComponent() {
- return flushingComponent;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.FLUSH;
- }
-
- @Override
- public int compareTo(LSMRTreeFlushOperation o) {
- return rtreeFlushTarget.getFile().getName().compareTo(o.getRTreeFlushTarget().getFile().getName());
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
index 847533f..9b3aa0c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeMergeOperation.java
@@ -18,114 +18,32 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
-public class LSMRTreeMergeOperation implements ILSMIOOperation {
- private final ILSMIndexAccessor accessor;
- private final List<ILSMComponent> mergingComponents;
- private final ITreeIndexCursor cursor;
- private final FileReference rtreeMergeTarget;
+public class LSMRTreeMergeOperation extends MergeOperation {
private final FileReference btreeMergeTarget;
private final FileReference bloomFilterMergeTarget;
- private final ILSMIOOperationCallback callback;
- private final String indexIdentifier;
- private boolean keepDeletedTuples;
public LSMRTreeMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
- ITreeIndexCursor cursor, FileReference rtreeMergeTarget, FileReference btreeMergeTarget,
+ ITreeIndexCursor cursor, FileReference target, FileReference btreeMergeTarget,
FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
- this.accessor = accessor;
- this.mergingComponents = mergingComponents;
- this.cursor = cursor;
- this.rtreeMergeTarget = rtreeMergeTarget;
+ super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
this.btreeMergeTarget = btreeMergeTarget;
this.bloomFilterMergeTarget = bloomFilterMergeTarget;
- this.callback = callback;
- this.indexIdentifier = indexIdentifier;
}
- @Override
- public Set<IODeviceHandle> getReadDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- for (ILSMComponent o : mergingComponents) {
- LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) o;
- devs.add(component.getRTree().getFileReference().getDeviceHandle());
- if (component.getBTree() != null) {
- devs.add(component.getBTree().getFileReference().getDeviceHandle());
- devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
- }
- }
- return devs;
- }
-
- @Override
- public Set<IODeviceHandle> getWriteDevices() {
- Set<IODeviceHandle> devs = new HashSet<>();
- devs.add(rtreeMergeTarget.getDeviceHandle());
- if (btreeMergeTarget != null) {
- devs.add(btreeMergeTarget.getDeviceHandle());
- devs.add(bloomFilterMergeTarget.getDeviceHandle());
- }
- return devs;
- }
-
- @Override
- public Boolean call() throws HyracksDataException {
- accessor.merge(this);
- return true;
- }
-
- @Override
- public ILSMIOOperationCallback getCallback() {
- return callback;
- }
-
- public FileReference getRTreeMergeTarget() {
- return rtreeMergeTarget;
- }
-
- public FileReference getBTreeMergeTarget() {
+ public FileReference getBTreeTarget() {
return btreeMergeTarget;
}
- public FileReference getBloomFilterMergeTarget() {
+ public FileReference getBloomFilterTarget() {
return bloomFilterMergeTarget;
}
-
- public ITreeIndexCursor getCursor() {
- return cursor;
- }
-
- public List<ILSMComponent> getMergingComponents() {
- return mergingComponents;
- }
-
- @Override
- public String getIndexUniqueIdentifier() {
- return indexIdentifier;
- }
-
- @Override
- public LSMIOOpertionType getIOOpertionType() {
- return LSMIOOpertionType.MERGE;
- }
-
- public boolean isKeepDeletedTuples() {
- return keepDeletedTuples;
- }
-
- public void setKeepDeletedTuples(boolean keepDeletedTuples) {
- this.keepDeletedTuples = keepDeletedTuples;
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
index 83ca78c..a66fa62 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeOpContext.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.rtree.impls;
-import java.util.LinkedList;
import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -28,9 +27,6 @@ import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
@@ -47,36 +43,22 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
private RTree.RTreeAccessor currentMutableRTreeAccessor;
private BTree.BTreeAccessor[] mutableBTreeAccessors;
private BTree.BTreeAccessor currentMutableBTreeAccessor;
-
private RTreeOpContext[] rtreeOpContexts;
private BTreeOpContext[] btreeOpContexts;
private RTreeOpContext currentRTreeOpContext;
private BTreeOpContext currentBTreeOpContext;
-
- private IndexOperation op;
- private final List<ILSMComponent> componentHolder;
- private final List<ILSMDiskComponent> componentsToBeMerged;
- private final List<ILSMDiskComponent> componentsToBeReplicated;
- private IModificationOperationCallback modificationCallback;
- private ISearchOperationCallback searchCallback;
- private final PermutingTupleReference indexTuple;
- private final MultiComparator filterCmp;
- private final PermutingTupleReference filterTuple;
- private ISearchPredicate searchPredicate;
private LSMRTreeCursorInitialState searchInitialState;
public LSMRTreeOpContext(List<ILSMMemoryComponent> mutableComponents, ITreeIndexFrameFactory rtreeLeafFrameFactory,
ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory btreeLeafFrameFactory,
IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
int[] rtreeFields, int[] filterFields, ILSMHarness lsmHarness, int[] comparatorFields,
- IBinaryComparatorFactory[] linearizerArray) {
+ IBinaryComparatorFactory[] linearizerArray, IBinaryComparatorFactory[] filterComparatorFactories) {
+ super(rtreeFields, filterFields, filterComparatorFactories, searchCallback, modificationCallback);
mutableRTreeAccessors = new RTree.RTreeAccessor[mutableComponents.size()];
mutableBTreeAccessors = new BTree.BTreeAccessor[mutableComponents.size()];
rtreeOpContexts = new RTreeOpContext[mutableComponents.size()];
btreeOpContexts = new BTreeOpContext[mutableComponents.size()];
-
- LSMRTreeMemoryComponent c = (LSMRTreeMemoryComponent) mutableComponents.get(0);
-
for (int i = 0; i < mutableComponents.size(); i++) {
LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) mutableComponents.get(i);
mutableRTreeAccessors[i] = (RTree.RTreeAccessor) mutableComponent.getRTree()
@@ -87,37 +69,14 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
rtreeOpContexts[i] = mutableRTreeAccessors[i].getOpContext();
btreeOpContexts[i] = mutableBTreeAccessors[i].getOpContext();
}
-
- assert mutableComponents.size() > 0;
currentRTreeOpContext = rtreeOpContexts[0];
currentBTreeOpContext = btreeOpContexts[0];
- this.componentHolder = new LinkedList<>();
- this.componentsToBeMerged = new LinkedList<>();
- this.componentsToBeReplicated = new LinkedList<>();
- this.modificationCallback = modificationCallback;
- this.searchCallback = searchCallback;
-
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
- filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
- } else {
- indexTuple = null;
- filterCmp = null;
- filterTuple = null;
- }
searchInitialState = new LSMRTreeCursorInitialState(rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
btreeLeafFrameFactory, getBTreeMultiComparator(), lsmHarness, comparatorFields, linearizerArray,
searchCallback, componentHolder);
}
@Override
- public void setOperation(IndexOperation newOp) {
- reset();
- this.op = newOp;
- }
-
- @Override
public void setCurrentMutableComponentId(int currentMutableComponentId) {
currentMutableRTreeAccessor = mutableRTreeAccessors[currentMutableComponentId];
currentMutableBTreeAccessor = mutableBTreeAccessors[currentMutableComponentId];
@@ -130,43 +89,11 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
}
}
- @Override
- public void reset() {
- super.reset();
- componentHolder.clear();
- componentsToBeMerged.clear();
- }
-
- @Override
- public IndexOperation getOperation() {
- return op;
- }
-
public MultiComparator getBTreeMultiComparator() {
return currentBTreeOpContext.getCmp();
}
@Override
- public List<ILSMComponent> getComponentHolder() {
- return componentHolder;
- }
-
- @Override
- public ISearchOperationCallback getSearchOperationCallback() {
- return searchCallback;
- }
-
- @Override
- public IModificationOperationCallback getModificationCallback() {
- return modificationCallback;
- }
-
- @Override
- public List<ILSMDiskComponent> getComponentsToBeMerged() {
- return componentsToBeMerged;
- }
-
- @Override
public void setSearchPredicate(ISearchPredicate searchPredicate) {
this.searchPredicate = searchPredicate;
}
@@ -176,23 +103,10 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
return searchPredicate;
}
- @Override
- public List<ILSMDiskComponent> getComponentsToBeReplicated() {
- return componentsToBeReplicated;
- }
-
- public MultiComparator getFilterCmp() {
- return filterCmp;
- }
-
public LSMRTreeCursorInitialState getSearchInitialState() {
return searchInitialState;
}
- public PermutingTupleReference getIndexTuple() {
- return indexTuple;
- }
-
public RTree.RTreeAccessor getCurrentMutableRTreeAccessor() {
return currentMutableRTreeAccessor;
}
@@ -201,11 +115,7 @@ public final class LSMRTreeOpContext extends AbstractLSMIndexOperationContext {
return currentMutableBTreeAccessor;
}
- public PermutingTupleReference getFilterTuple() {
- return filterTuple;
- }
-
public RTreeOpContext getCurrentRTreeOpContext() {
return currentRTreeOpContext;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 018e6d3..bb85696 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -26,7 +26,6 @@ import java.util.Set;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -36,33 +35,36 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor.ICursorFactory;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.rtree.impls.RTree;
import org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
@@ -70,6 +72,7 @@ import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
+ private static final ICursorFactory cursorFactory = opCtx -> new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx);
// On-disk components.
// For creating RTree's used in bulk load. Different from diskRTreeFactory
// because it should have a different tuple writer in it's leaf frames.
@@ -98,71 +101,26 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
}
@Override
- public synchronized void activate() throws HyracksDataException {
- super.activate();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- immutableComponents.clear();
- List<LSMComponentFileReferences> validFileReferences;
- validFileReferences = fileManager.cleanupAndGetValidFiles();
- for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
- LSMRTreeDiskComponent component;
- component = createDiskComponent(componentFactory, lsmComonentFileReference.getInsertIndexFileReference(),
- null, null, false);
- immutableComponents.add(component);
- }
- isActivated = true;
- }
-
- @Override
- public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
- super.deactivate(flushOnExit);
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.deactivateCloseHandle();
- }
- isActivated = false;
+ protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
+ return createDiskComponent(componentFactory, refs.getInsertIndexFileReference(), null, null, false);
}
@Override
- public synchronized void deactivate() throws HyracksDataException {
- deactivate(true);
+ protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
+ rtree.deactivateCloseHandle();
}
@Override
- public synchronized void destroy() throws HyracksDataException {
- super.destroy();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.destroy();
- }
- fileManager.deleteDirs();
+ protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ ((LSMRTreeDiskComponent) c).getRTree().destroy();
}
@Override
- public synchronized void clear() throws HyracksDataException {
- super.clear();
- List<ILSMDiskComponent> immutableComponents = diskComponents;
- for (ILSMDiskComponent c : immutableComponents) {
- RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
- rtree.deactivate();
- rtree.destroy();
- }
- immutableComponents.clear();
- }
-
- @Override
- public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
- LSMRTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
- opCtx.setOperation(IndexOperation.FLUSH);
- opCtx.getComponentHolder().add(flushingComponent);
- ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, opCtx);
- ioScheduler.scheduleOperation(new LSMRTreeFlushOperation(accessor, flushingComponent,
- relFlushFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
+ protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+ RTree rtree = ((LSMRTreeDiskComponent) c).getRTree();
+ rtree.deactivate();
+ rtree.destroy();
}
@Override
@@ -177,8 +135,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
- LSMRTreeDiskComponent component =
- createDiskComponent(componentFactory, flushOp.getRTreeFlushTarget(), null, null, true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
RTree diskRTree = component.getRTree();
// scan the memory BTree
@@ -248,8 +205,8 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
flushingComponent.getMetadata().copy(component.getMetadata());
rTreeBulkloader.end();
@@ -258,35 +215,16 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
}
@Override
- public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
- throws HyracksDataException {
- LSMRTreeOpContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
- rctx.setOperation(IndexOperation.MERGE);
- List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
- boolean returnDeletedTuples = false;
- if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents
- .get(diskComponents.size() - 1)) {
- returnDeletedTuples = true;
- }
- ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(rctx, returnDeletedTuples);
- LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
- ILSMIndexAccessor accessor = new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness, rctx);
- ioScheduler.scheduleOperation(new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
- relMergeFileRefs.getInsertIndexFileReference(), null, null, callback, fileManager.getBaseDir()));
- }
-
- @Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
- LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
- ITreeIndexCursor cursor = mergeOp.getCursor();
+ MergeOperation mergeOp = (MergeOperation) operation;
+ IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
// Bulk load the tuples from all on-disk RTrees into the new RTree.
- LSMRTreeDiskComponent component =
- createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(), null, null, true);
+ LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
RTree mergedRTree = component.getRTree();
IIndexBulkLoader bulkloader = mergedRTree.createBulkLoader(1.0f, false, 0L, false);
try {
@@ -304,8 +242,8 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
- filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
- filterManager.writeFilter(component.getLSMComponentFilter(), component.getRTree());
+ getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+ getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
bulkloader.end();
@@ -315,30 +253,14 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
@Override
public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
ISearchOperationCallback searchCallback) {
- return new LSMRTreeWithAntiMatterTuplesAccessor(lsmHarness,
- createOpContext(modificationCallback, searchCallback));
- }
-
- public class LSMRTreeWithAntiMatterTuplesAccessor extends LSMTreeIndexAccessor {
- public LSMRTreeWithAntiMatterTuplesAccessor(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) {
- super(lsmHarness, ctx);
- }
-
- @Override
- public ITreeIndexCursor createSearchCursor(boolean exclusive) {
- return new LSMRTreeWithAntiMatterTuplesSearchCursor(ctx);
- }
-
- public MultiComparator getMultiComparator() {
- LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) ctx;
- return concreteCtx.getCurrentRTreeOpContext().getCmp();
- }
+ LSMRTreeOpContext opCtx = createOpContext(modificationCallback, searchCallback);
+ return new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
}
@Override
- public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- return new LSMRTreeWithAntiMatterTuplesBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex);
+ public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
+ return new LSMRTreeWithAntiMatterTuplesBulkLoader(fillLevel, verifyInput, numElementsHint);
}
public class LSMRTreeWithAntiMatterTuplesBulkLoader implements IIndexBulkLoader {
@@ -350,11 +272,8 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
public final PermutingTupleReference filterTuple;
public final MultiComparator filterCmp;
- public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex) throws HyracksDataException {
- if (checkIfEmptyIndex && !isEmptyIndex()) {
- throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
- }
+ public LSMRTreeWithAntiMatterTuplesBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+ throws HyracksDataException {
// Note that by using a flush target file name, we state that the
// new bulk loaded tree is "newer" than any other merged tree.
@@ -362,10 +281,10 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
bulkLoader = ((LSMRTreeDiskComponent) component).getRTree().createBulkLoader(fillFactor, verifyInput,
numElementsHint, false);
- if (filterFields != null) {
- indexTuple = new PermutingTupleReference(rtreeFields);
+ if (getFilterFields() != null) {
+ indexTuple = new PermutingTupleReference(getTreeFields());
filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
- filterTuple = new PermutingTupleReference(filterFields);
+ filterTuple = new PermutingTupleReference(getFilterFields());
} else {
indexTuple = null;
filterCmp = null;
@@ -405,7 +324,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
if (!cleanedUpArtifacts) {
if (component.getLSMComponentFilter() != null) {
- filterManager.writeFilter(component.getLSMComponentFilter(),
+ getFilterManager().writeFilter(component.getLSMComponentFilter(),
((LSMRTreeDiskComponent) component).getRTree());
}
bulkLoader.end();
@@ -414,7 +333,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
cleanupArtifacts();
} else {
ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
- lsmHarness.addBulkLoadedComponent(component);
+ getLsmHarness().addBulkLoadedComponent(component);
}
}
}
@@ -455,4 +374,27 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree {
files.add(rtree.getFileReference().getFile().getAbsolutePath());
return files;
}
+
+ @Override
+ protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+ ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
+ return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+ null, null, callback, fileManager.getBaseDir());
+ }
+
+ @Override
+ protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+ List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+ ILSMIOOperationCallback callback) throws HyracksDataException {
+ boolean returnDeletedTuples = false;
+ if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
+ returnDeletedTuples = true;
+ }
+ ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx, returnDeletedTuples);
+ ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
+ return new MergeOperation(accessor, mergeFileRefs.getInsertIndexFileReference(), callback,
+ fileManager.getBaseDir(), mergingComponents, cursor);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
index cc34a89..af9fef6 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuplesFlushCursor.java
@@ -27,7 +27,6 @@ import org.apache.hyracks.storage.common.ICursorInitialState;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ITreeIndexCursor {
private final TreeTupleSorter rTreeTupleSorter;
@@ -155,11 +154,6 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ITreeIndexCursor
}
@Override
- public ICachedPage getPage() {
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
}
@@ -170,12 +164,7 @@ public class LSMRTreeWithAntiMatterTuplesFlushCursor implements ITreeIndexCursor
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
index 724ddcf..cad1f74 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/TreeTupleSorter.java
@@ -75,8 +75,8 @@ public class TreeTupleSorter implements ITreeIndexCursor {
// We don't latch pages since this code is only used by flush () before
// bulk-loading the r-tree to disk and flush is not concurrent.
//
- ICachedPage node1 = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, tPointers[currentTupleIndex * 2]),
- false);
+ ICachedPage node1 =
+ bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, tPointers[currentTupleIndex * 2]), false);
try {
leafFrame1.setPage(node1);
frameTuple1.resetByTupleOffset(leafFrame1.getBuffer().array(), tPointers[currentTupleIndex * 2 + 1]);
@@ -222,11 +222,6 @@ public class TreeTupleSorter implements ITreeIndexCursor {
}
@Override
- public ICachedPage getPage() {
- return null;
- }
-
- @Override
public void setBufferCache(IBufferCache bufferCache) {
// do nothing
}
@@ -237,12 +232,7 @@ public class TreeTupleSorter implements ITreeIndexCursor {
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index bb3d2f5..4302c61 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -97,11 +97,6 @@ public class RTreeSearchCursor implements ITreeIndexCursor {
return pageId;
}
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
protected boolean fetchNextLeafPage() throws HyracksDataException {
boolean succeeded = false;
if (readLatched) {
@@ -114,8 +109,12 @@ public class RTreeSearchCursor implements ITreeIndexCursor {
int pageId = pathList.getLastPageId();
long parentLsn = pathList.getLastPageLsn();
pathList.moveLast();
- if(pageId <0) throw new IllegalStateException();
- if(fileId<0) throw new IllegalStateException();
+ if (pageId < 0) {
+ throw new IllegalStateException();
+ }
+ if (fileId < 0) {
+ throw new IllegalStateException();
+ }
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
node.acquireReadLatch();
readLatched = true;
@@ -265,12 +264,7 @@ public class RTreeSearchCursor implements ITreeIndexCursor {
}
@Override
- public boolean exclusiveLatchNodes() {
+ public boolean isExclusiveLatchNodes() {
return false;
}
-
- @Override
- public void markCurrentTupleAsUpdated() throws HyracksDataException {
- throw new HyracksDataException("Updating tuples is not supported with this cursor.");
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
index b23a345..255e0d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/IIndex.java
@@ -101,18 +101,20 @@ public interface IIndex {
ISearchOperationCallback searchCallback) throws HyracksDataException;
/**
+ * Strictly a test method
+ *
* Ensures that all pages (and tuples) of the index are logically consistent.
* An assertion error is thrown if validation fails.
*
* @throws HyracksDataException
* if there is an error performing validation
*/
- public void validate() throws HyracksDataException;
+ void validate() throws HyracksDataException;
/**
* @return the {@link IBufferCache} underlying this index.
*/
- public IBufferCache getBufferCache();
+ IBufferCache getBufferCache();
/**
* @return the size, in bytes, of pre-allocated memory space that this index was allotted.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
index 4848711..024b72d 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/multithread/LSMBTreeTestWorker.java
@@ -30,7 +30,8 @@ import org.apache.hyracks.storage.am.common.TestOperationSelector;
import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree;
-import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree.LSMBTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeOpContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexCursor;
@@ -52,9 +53,10 @@ public class LSMBTreeTestWorker extends AbstractIndexTestWorker {
@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
- LSMBTreeAccessor accessor = (LSMBTreeAccessor) indexAccessor;
+ LSMTreeIndexAccessor accessor = (LSMTreeIndexAccessor) indexAccessor;
IIndexCursor searchCursor = accessor.createSearchCursor(false);
- MultiComparator cmp = accessor.getMultiComparator();
+ LSMBTreeOpContext concreteCtx = (LSMBTreeOpContext) accessor.getCtx();
+ MultiComparator cmp = concreteCtx.getCmp();
RangePredicate rangePred = new RangePredicate(tuple, tuple, true, true, cmp, cmp);
switch (op) {