You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/11/24 06:30:10 UTC
[06/12] asterixdb git commit: Cleanup FileSplit and FileReference
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index 44778a2..c089854 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -73,7 +73,8 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
protected PermutingFrameTupleReference maxFilterKey;
public IndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- IRecordDescriptorProvider recordDescProvider, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) {
+ IRecordDescriptorProvider recordDescProvider, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes)
+ throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition);
@@ -133,7 +134,7 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput
dos = tb.getDataOutput();
appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(indexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, null);
indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
cursor = createCursor();
if (retainInput) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
index 0823a74..30c7197 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -61,7 +62,7 @@ public class TreeIndexBulkLoadOperatorDescriptor extends AbstractTreeIndexOperat
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, fillFactor, verifyInput,
numElementsHint, checkIfEmptyIndex, recordDescProvider);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
index e3d00d5..e491d1d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -49,7 +50,7 @@ public class TreeIndexCreateOperatorDescriptor extends AbstractTreeIndexOperator
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexCreateOperatorNodePushable(this, ctx, partition);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
index 6d3d7a1..07cab2e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
@@ -28,7 +28,7 @@ import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
public abstract class TreeIndexDataflowHelper extends IndexDataflowHelper {
public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- boolean durable) {
+ boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, durable);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
index d3f1723..8589290 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -47,7 +48,7 @@ public class TreeIndexDiskOrderScanOperatorDescriptor extends AbstractTreeIndexO
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new TreeIndexDiskOrderScanOperatorNodePushable(this, ctx, partition);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
index 3ac6065..fc47548 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
@@ -34,6 +34,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final AbstractTreeIndexOperatorDescriptor opDesc;
@@ -42,7 +43,7 @@ public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOut
private ITreeIndex treeIndex;
public TreeIndexDiskOrderScanOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition) {
+ IHyracksTaskContext ctx, int partition) throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
@@ -56,8 +57,9 @@ public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOut
try {
ITreeIndexFrame cursorFrame = treeIndex.getLeafFrameFactory().createFrame();
ITreeIndexCursor cursor = treeIndexHelper.createDiskOrderScanCursor(cursorFrame);
+ LocalResource resource = treeIndexHelper.getResource();
ISearchOperationCallback searchCallback = opDesc.getSearchOpCallbackFactory()
- .createSearchOperationCallback(treeIndexHelper.getResourceID(), ctx, null);
+ .createSearchOperationCallback(resource.getId(), ctx, null);
ITreeIndexAccessor indexAccessor = (ITreeIndexAccessor) treeIndex
.createAccessor(NoOpOperationCallback.INSTANCE, searchCallback);
try {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
index 128be56..6b55b5e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -58,7 +59,7 @@ public class TreeIndexInsertUpdateDeleteOperatorDescriptor extends AbstractTreeI
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexInsertUpdateDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
recordDescProvider, op);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
index 1e9b75e..06371d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
@@ -52,7 +53,7 @@ public class TreeIndexStatsOperatorDescriptor extends AbstractTreeIndexOperatorD
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new TreeIndexStatsOperatorNodePushable(this, ctx, partition);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index f70df29..5121f89 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -24,6 +24,8 @@ import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
@@ -33,6 +35,7 @@ import org.apache.hyracks.storage.am.common.util.TreeIndexStats;
import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
+import org.apache.hyracks.storage.common.file.LocalResource;
public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
private final AbstractTreeIndexOperatorDescriptor opDesc;
@@ -42,7 +45,7 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc
private TreeIndexStatsGatherer statsGatherer;
public TreeIndexStatsOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
this.opDesc = opDesc;
this.ctx = ctx;
this.treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
@@ -67,7 +70,10 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc
writer.open();
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- int indexFileId = fileMapProvider.lookupFileId(treeIndexHelper.getFileReference());
+ LocalResource resource = treeIndexHelper.getResource();
+ IIOManager ioManager = ctx.getIOManager();
+ FileReference fileRef = ioManager.getFileRef(resource.getPath(), true);
+ int indexFileId = fileMapProvider.lookupFileId(fileRef);
statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getMetaManager(), indexFileId,
treeIndex.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
index c2cca41..90eb5a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/NoOpOperationCallbackFactory.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
+import org.apache.hyracks.storage.common.file.LocalResource;
/**
* Dummy NoOp callback factory used primarily for testing. Always returns the {@link NoOpOperationCallback} instance.
@@ -40,9 +41,8 @@ public enum NoOpOperationCallbackFactory
}
@Override
- public IModificationOperationCallback createModificationOperationCallback(String resourcePath, long resourceId,
- int resourcePartition, Object resource, IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable)
- throws HyracksDataException {
+ public IModificationOperationCallback createModificationOperationCallback(LocalResource resource,
+ IHyracksTaskContext ctx, IOperatorNodePushable operatorNodePushable) throws HyracksDataException {
return NoOpOperationCallback.INSTANCE;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
index 50f3e68..496a2a2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/IndexFileNameUtil.java
@@ -21,7 +21,9 @@ package org.apache.hyracks.storage.am.common.util;
import java.io.File;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
@@ -34,9 +36,9 @@ public class IndexFileNameUtil {
return path + File.separator + IO_DEVICE_NAME_PREFIX + ioDeviceId;
}
- public static FileReference getIndexAbsoluteFileRef(IIndexOperatorDescriptor opDesc, int partition, IIOManager ioManager){
- String indexName = opDesc.getFileSplitProvider().getFileSplits()[partition].getLocalFile().getFile().getPath();
- int ioDeviceId = opDesc.getFileSplitProvider().getFileSplits()[partition].getIODeviceId();
- return ioManager.getAbsoluteFileRef(ioDeviceId, indexName);
+ public static FileReference getIndexAbsoluteFileRef(IIndexOperatorDescriptor opDesc, int partition,
+ IIOManager ioManager) throws HyracksDataException {
+ FileSplit split = opDesc.getFileSplitProvider().getFileSplits()[partition];
+ return ioManager.getFileRef(split.getPath(), split.isManaged());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index ef8f646..9d9ed23 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -18,20 +18,19 @@
*/
package org.apache.hyracks.storage.am.lsm.btree.dataflow;
-import java.util.List;
-
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerProvider;
-import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
@@ -40,26 +39,19 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int version, boolean durable)
+ throws HyracksDataException {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, false, null, null, null, null, durable);
this.version = version;
}
- public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
- List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
- ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version, boolean durable) {
- this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version, durable);
- }
-
@Override
public IIndex getIndexInstance() {
synchronized (lcManager) {
if (index == null) {
try {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceName);
} catch (HyracksDataException e) {
return null;
}
@@ -71,7 +63,10 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTree(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createExternalBTree(ctx.getIOManager(), fileRef, opDesc.getStorageManager().getBufferCache(
+ ctx), opDesc
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
index a4a0792..6334fb3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -46,10 +47,10 @@ public class ExternalBTreeDataflowHelperFactory extends AbstractLSMIndexDataflow
@Override
public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new ExternalBTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version, durable);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, version, durable);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index e7a1161..95fdbf1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -20,9 +20,11 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.IIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -38,7 +40,7 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- int[] buddyBtreeFields, int version, boolean durable) {
+ int[] buddyBtreeFields, int version, boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, null,
null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
@@ -48,7 +50,8 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version, boolean durable) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version, boolean durable)
+ throws HyracksDataException {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
ioOpCallbackFactory, null, null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
@@ -60,7 +63,7 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
synchronized (lcManager) {
if (index == null) {
try {
- index = lcManager.get(resourcePath);
+ index = lcManager.get(resourceName);
} catch (HyracksDataException e) {
return null;
}
@@ -72,11 +75,14 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
@Override
protected IIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createExternalBTreeWithBuddy(file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
- .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
- .getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory
- .getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(),
- buddyBtreeFields, version, durable);
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createExternalBTreeWithBuddy(ctx.getIOManager(), fileRef, opDesc.getStorageManager()
+ .getBufferCache(ctx),
+ opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
+ treeOpDesc.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy,
+ opTrackerFactory.getOperationTracker(ctx), ioScheduler,
+ ioOpCallbackFactory.createIOOperationCallback(), buddyBtreeFields, version, durable);
}
public int getTargetVersion() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
index acf502f..319b5c1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.btree.dataflow;
import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -47,7 +48,7 @@ public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMInde
@Override
public IIndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new ExternalBTreeWithBuddyDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version, durable);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index 6b2f5b1..25468a3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -25,9 +25,11 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -46,7 +48,7 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
- int[] filterFields, boolean durable) {
+ int[] filterFields, boolean durable) throws HyracksDataException {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
filterCmpFactories, btreeFields, filterFields, durable);
@@ -57,7 +59,7 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields, boolean durable) {
+ int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.needKeyDupCheck = needKeyDupCheck;
@@ -67,7 +69,10 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx),
+ FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(treeOpDesc, ctx.getTaskAttemptId()
+ .getTaskId().getPartition(), ctx.getIOManager());
+ return LSMBTreeUtils.createLSMTree(ctx.getIOManager(), virtualBufferCaches, fileRef, opDesc.getStorageManager()
+ .getBufferCache(ctx),
opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index 63b784a..4093938 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -54,7 +55,7 @@ public class LSMBTreeDataflowHelperFactory extends AbstractLSMIndexDataflowHelpe
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
- int partition) {
+ int partition) throws HyracksDataException {
return new LSMBTreeDataflowHelper(opDesc, ctx, partition,
virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 1175130..ca269de 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -89,7 +90,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
private final ITreeIndexFrameFactory interiorFrameFactory;
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
- public ExternalBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ public ExternalBTree(IIOManager ioManager, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
@@ -97,12 +99,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version,
boolean durable) {
- super(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
+ super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager,
+ diskBTreeFactory,
bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory,
null);
- this.secondDiskComponents = new LinkedList<ILSMComponent>();
+ this.secondDiskComponents = new LinkedList<>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index fd60b6d..6181299 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
@@ -94,7 +95,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
private final List<ILSMComponent> secondDiskComponents;
private int version = -1;
- public ExternalBTreeWithBuddy(ITreeIndexFrameFactory btreeInteriorFrameFactory,
+ public ExternalBTreeWithBuddy(IIOManager ioManager, ITreeIndexFrameFactory btreeInteriorFrameFactory,
ITreeIndexFrameFactory btreeLeafFrameFactory, ITreeIndexFrameFactory buddyBtreeLeafFrameFactory,
IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
@@ -103,7 +104,8 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
int[] buddyBTreeFields, int version, boolean durable) {
- super(diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
+ super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+ opTracker,
ioScheduler, ioOpCallback, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
@@ -115,7 +117,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
bloomFilterFactory);
this.bulkComponentFactory = new LSMBTreeWithBuddyDiskComponentFactory(bulkLoadBTreeFactory, buddyBtreeFactory,
bloomFilterFactory);
- this.secondDiskComponents = new LinkedList<ILSMComponent>();
+ this.secondDiskComponents = new LinkedList<>();
this.version = version;
}
@@ -876,7 +878,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
files.add(component.getBuddyBTree().getFileReference().getFile().getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 6d673a9..3f05399 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.btree.impls;
-import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -28,6 +27,7 @@ import java.util.Set;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
@@ -100,7 +100,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
// Primary LSMBTree has a Bloomfilter, but Secondary one doesn't have.
private final boolean hasBloomFilter;
- public LSMBTree(List<IVirtualBufferCache> virtualBufferCaches, ITreeIndexFrameFactory interiorFrameFactory,
+ public LSMBTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
@@ -109,8 +110,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
- boolean durable) {
- super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ boolean durable) throws HyracksDataException {
+ super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
filterManager, filterFields, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
@@ -122,7 +123,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), interiorFrameFactory,
insertLeafFrameFactory, cmpFactories, fieldCount,
- new FileReference(new File(fileManager.getBaseDir() + "_virtual_" + i))),
+ ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_" + i, false)),
virtualBufferCache, i == 0 ? true : false,
filterFactory == null ? null : filterFactory.createLSMComponentFilter());
memoryComponents.add(mutableComponent);
@@ -137,14 +138,16 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
// Without memory components
- public LSMBTree(ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory insertLeafFrameFactory,
+ public LSMBTree(IIOManager ioManager,
+ ITreeIndexFrameFactory insertLeafFrameFactory,
ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
- super(diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
+ super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+ bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
@@ -483,7 +486,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
if (component.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
@@ -565,7 +568,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
}
if (mergedComponent.getLSMComponentFilter() != null) {
- List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
+ List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
@@ -882,7 +885,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
@Override
public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
- Set<String> files = new HashSet<String>();
+ Set<String> files = new HashSet<>();
LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
files.add(component.getBTree().getFileReference().getFile().getAbsolutePath());
if (hasBloomFilter) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index f62f70c..bf6ef24 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -32,6 +32,7 @@ import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -45,20 +46,20 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final boolean hasBloomFilter;
- public LSMBTreeFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory, boolean hasBloomFilter) {
- super(fileMapProvider, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.btreeFactory = btreeFactory;
this.hasBloomFilter = hasBloomFilter;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false) : null);
}
@Override
@@ -69,8 +70,8 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false) : null);
}
private static FilenameFilter btreeFilter = new FilenameFilter() {
@@ -82,9 +83,9 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// create transaction filter <to hide transaction files>
FilenameFilter transactionFilter = getTransactionFileFilter(false);
@@ -93,7 +94,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
// List of valid BTree files.
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
+ HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -127,14 +128,14 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
Collections.sort(allBloomFilterFiles);
}
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
validComparableBTreeFiles.add(lastBTree);
List<ComparableFileName> validComparableBloomFilterFiles = null;
ComparableFileName lastBloomFilter = null;
if (hasBloomFilter) {
- validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ validComparableBloomFilterFiles = new ArrayList<>();
lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
}
@@ -204,8 +205,8 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a transaction
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
- createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false), null,
+ createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING, false));
}
@Override
@@ -244,8 +245,8 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
throw new HyracksDataException("unrecognized file found = " + fileName);
}
}
- FileReference bTreeFileRef = new FileReference(bTreeFile);
- FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+ FileReference bTreeFileRef = ioManager.getFileRef(bTreeFile.getAbsolutePath(), false);
+ FileReference bloomFilterFileRef = ioManager.getFileRef(bloomFilterFile.getAbsolutePath(), false);
return new LSMComponentFileReferences(bTreeFileRef, null, bloomFilterFileRef);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index ed39630..e0bebf4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -31,6 +31,7 @@ import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.common.api.IndexException;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
@@ -59,22 +60,23 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
}
};
- public LSMBTreeWithBuddyFileManager(IFileMapProvider fileMapProvider, FileReference file,
+ public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory,
TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) {
- super(fileMapProvider, file, null);
+ super(ioManager, fileMapProvider, file, null);
this.buddyBtreeFactory = buddyBtreeFactory;
this.btreeFactory = btreeFactory;
}
@Override
- public LSMComponentFileReferences getRelFlushFileReference() {
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String ts = getCurrentTimestamp();
String baseName = baseDir + ts + SPLIT_STRING + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createFlushFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -86,17 +88,18 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
- createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createMergeFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createMergeFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException, IndexException {
- List<LSMComponentFileReferences> validFiles = new ArrayList<LSMComponentFileReferences>();
- ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<ComparableFileName>();
- ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<LSMComponentFileReferences> validFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<>();
+ ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
// Create transaction file filter
FilenameFilter transactionFilefilter = getTransactionFileFilter(false);
@@ -104,7 +107,7 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
// Gather files.
cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory,
allBTreeFiles);
- HashSet<String> btreeFilesSet = new HashSet<String>();
+ HashSet<String> btreeFilesSet = new HashSet<>();
for (ComparableFileName cmpFileName : allBTreeFiles) {
int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
@@ -136,15 +139,15 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
Collections.sort(allBuddyBTreeFiles);
Collections.sort(allBloomFilterFiles);
- List<ComparableFileName> validComparableBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBTreeFiles = new ArrayList<>();
ComparableFileName lastBTree = allBTreeFiles.get(0);
validComparableBTreeFiles.add(lastBTree);
- List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBuddyBTreeFiles = new ArrayList<>();
ComparableFileName lastBuddyBTree = allBuddyBTreeFiles.get(0);
validComparableBuddyBTreeFiles.add(lastBuddyBTree);
- List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<ComparableFileName>();
+ List<ComparableFileName> validComparableBloomFilterFiles = new ArrayList<>();
ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0);
validComparableBloomFilterFiles.add(lastBloomFilter);
@@ -208,9 +211,10 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
String baseName = baseDir + ts + SPLIT_STRING + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
- createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), createFlushFile(baseName + SPLIT_STRING
- + BLOOM_FILTER_STRING));
+ return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING, false),
+ createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING, false), createFlushFile(baseName
+ + SPLIT_STRING
+ + BLOOM_FILTER_STRING, false));
}
@Override
@@ -252,9 +256,9 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
throw new HyracksDataException("unrecognized file found = " + fileName);
}
}
- FileReference bTreeFileRef = new FileReference(bTreeFile);
- FileReference buddyBTreeFileRef = new FileReference(buddyBTreeFile);
- FileReference bloomFilterFileRef = new FileReference(bloomFilterFile);
+ FileReference bTreeFileRef = ioManager.getFileRef(bTreeFile.getAbsolutePath(), false);
+ FileReference buddyBTreeFileRef = ioManager.getFileRef(buddyBTreeFile.getAbsolutePath(), false);
+ FileReference bloomFilterFileRef = ioManager.getFileRef(bloomFilterFile.getAbsolutePath(), false);
return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index 16901e9..38a3a7b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -23,7 +23,9 @@ import java.util.List;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
@@ -57,12 +59,14 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.file.IFileMapProvider;
public class LSMBTreeUtils {
- public static LSMBTree createLSMTree(List<IVirtualBufferCache> virtualBufferCaches, FileReference file,
+ public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file,
IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable)
+ throws HyracksDataException {
LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
cmpFactories.length, false);
LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits,
@@ -77,10 +81,10 @@ public class LSMBTreeUtils {
IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = needKeyDupCheck
@@ -98,10 +102,11 @@ public class LSMBTreeUtils {
}
//Primary LSMBTree index has a BloomFilter.
- ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskBTreeFactory,
needKeyDupCheck);
- LSMBTree lsmTree = new LSMBTree(virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
+ LSMBTree lsmTree = new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider,
typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck,
@@ -109,7 +114,8 @@ public class LSMBTreeUtils {
return lsmTree;
}
- public static ExternalBTree createExternalBTree(FileReference file, IBufferCache diskBufferCache,
+ public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
+ IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
@@ -134,32 +140,36 @@ public class LSMBTreeUtils {
// can be used for both inserts and deletes
ITreeIndexFrameFactory dualLeafFrameFactory = new BTreeNSMLeafFrameFactory(referencingTupleWriterFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider,
bloomFilterKeyFields);
// This is the component factory for transactions
- TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, dualLeafFrameFactory, cmpFactories, typeTraits.length);
//TODO remove BloomFilter from external dataset's secondary LSMBTree index
- ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(diskFileMapProvider, file, diskBTreeFactory,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file,
+ diskBTreeFactory,
true);
// the disk only index uses an empty ArrayList for virtual buffer caches
- ExternalBTree lsmTree = new ExternalBTree(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
+ ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory,
+ deleteLeafFrameFactory,
fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, startWithVersion, durable);
return lsmTree;
}
- public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(FileReference file, IBufferCache diskBufferCache,
+ public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file,
+ IBufferCache diskBufferCache,
IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
@@ -189,11 +199,11 @@ public class LSMBTreeUtils {
IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
metaFrameFactory);
- TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories,
typeTraits.length);
- TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
@@ -204,15 +214,17 @@ public class LSMBTreeUtils {
bloomFilterKeyFields);
// buddy b-tree factory
- TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
+ TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+ diskFileMapProvider,
freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory,
buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
- ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(diskFileMapProvider, file,
+ ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file,
diskBTreeFactory, diskBuddyBTreeFactory);
// the disk only index uses an empty ArrayList for virtual buffer caches
- ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(interiorFrameFactory, insertLeafFrameFactory,
+ ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory,
+ insertLeafFrameFactory,
buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, bulkLoadBTreeFactory, diskBTreeFactory,
diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate,
mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index e97913f..a464d96 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -39,7 +39,7 @@ public interface ILSMIndexFileManager {
public void deleteDirs();
- public LSMComponentFileReferences getRelFlushFileReference();
+ public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
throws HyracksDataException;