You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/05/20 03:25:20 UTC
[3/4] incubator-asterixdb-hyracks git commit: Add a flag for
LSM-based indices to indicate whether force pages to disk devices during
flush and merge.
Add a flag for LSM-based indices to indicate whether force pages to disk devices during flush and merge.
Change-Id: I988716c03cffe30b008e144d3a478ee25e367212
Reviewed-on: https://asterix-gerrit.ics.uci.edu/240
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <ki...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/7f7c5730
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/7f7c5730
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/7f7c5730
Branch: refs/heads/master
Commit: 7f7c5730c6528c4a0c589cabc47447e9fa15d43b
Parents: 9b9f7c5
Author: buyingyi <bu...@gmail.com>
Authored: Mon May 18 21:06:45 2015 -0700
Committer: Ian Maxon <im...@uci.edu>
Committed: Tue May 19 17:24:19 2015 -0700
----------------------------------------------------------------------
.../algebra/metadata/IMetadataProvider.java | 10 ++---
.../hyracks/dataflow/std/file/FileSplit.java | 5 +++
.../btree/client/InsertPipelineExample.java | 6 +--
.../client/PrimaryIndexBulkLoadExample.java | 2 +-
.../btree/client/PrimaryIndexSearchExample.java | 2 +-
.../client/SecondaryIndexBulkLoadExample.java | 2 +-
.../client/SecondaryIndexSearchExample.java | 2 +-
.../tests/am/btree/BTreeOperatorTestHelper.java | 2 +-
.../AbstractfWordInvertedIndexTest.java | 2 +-
.../PartitionedWordInvertedIndexTest.java | 2 +-
.../am/invertedindex/WordInvertedIndexTest.java | 2 +-
.../lsm/btree/LSMBTreeOperatorTestHelper.java | 2 +-
.../lsm/rtree/LSMRTreeOperatorTestHelper.java | 2 +-
...eWithAntiMatterTuplesOperatorTestHelper.java | 2 +-
.../am/rtree/AbstractRTreeOperatorTest.java | 2 +-
.../tests/am/rtree/RTreeOperatorTestHelper.java | 4 +-
.../RTreeSecondaryIndexInsertOperatorTest.java | 3 +-
.../RTreeSecondaryIndexScanOperatorTest.java | 3 +-
.../RTreeSecondaryIndexSearchOperatorTest.java | 3 +-
.../RTreeSecondaryIndexStatsOperatorTest.java | 3 +-
.../am/btree/dataflow/BTreeDataflowHelper.java | 6 ++-
.../dataflow/BTreeDataflowHelperFactory.java | 8 +++-
.../am/common/dataflow/IndexDataflowHelper.java | 13 ++++++-
.../dataflow/TreeIndexDataflowHelper.java | 5 ++-
.../dataflow/ExternalBTreeDataflowHelper.java | 14 ++++---
.../ExternalBTreeDataflowHelperFactory.java | 8 ++--
.../ExternalBTreeWithBuddyDataflowHelper.java | 15 ++++----
...rnalBTreeWithBuddyDataflowHelperFactory.java | 8 ++--
.../btree/dataflow/LSMBTreeDataflowHelper.java | 10 ++---
.../dataflow/LSMBTreeDataflowHelperFactory.java | 6 +--
.../am/lsm/btree/impls/ExternalBTree.java | 39 +++++++++++---------
.../lsm/btree/impls/ExternalBTreeWithBuddy.java | 35 ++++++++++--------
.../storage/am/lsm/btree/impls/LSMBTree.java | 18 ++++-----
.../am/lsm/btree/util/LSMBTreeUtils.java | 12 +++---
.../am/lsm/common/api/ILSMIndexInternal.java | 10 ++++-
.../AbstractLSMIndexDataflowHelper.java | 10 +++--
.../AbstractLSMIndexDataflowHelperFactory.java | 5 ++-
.../common/impls/AbstractDiskLSMComponent.java | 2 +-
.../am/lsm/common/impls/AbstractLSMIndex.java | 21 ++++++++---
.../impls/AbstractMemoryLSMComponent.java | 1 -
.../lsm/common/impls/ExternalIndexHarness.java | 20 +++++-----
.../storage/am/lsm/common/impls/LSMHarness.java | 3 --
.../LSMInvertedIndexDataflowHelper.java | 12 +++---
.../LSMInvertedIndexDataflowHelperFactory.java | 8 ++--
...rtitionedLSMInvertedIndexDataflowHelper.java | 12 +++---
...edLSMInvertedIndexDataflowHelperFactory.java | 8 ++--
.../invertedindex/impls/LSMInvertedIndex.java | 12 +++---
.../impls/LSMInvertedIndexAccessor.java | 2 +
.../impls/PartitionedLSMInvertedIndex.java | 5 ++-
.../invertedindex/util/InvertedIndexUtils.java | 10 +++--
.../AbstractLSMRTreeDataflowHelper.java | 8 ++--
.../dataflow/ExternalRTreeDataflowHelper.java | 13 ++++---
.../ExternalRTreeDataflowHelperFactory.java | 9 +++--
.../rtree/dataflow/LSMRTreeDataflowHelper.java | 12 +++---
.../dataflow/LSMRTreeDataflowHelperFactory.java | 6 +--
...RTreeWithAntiMatterTuplesDataflowHelper.java | 12 +++---
...thAntiMatterTuplesDataflowHelperFactory.java | 7 ++--
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 26 ++++++-------
.../am/lsm/rtree/impls/ExternalRTree.java | 31 +++++++++-------
.../storage/am/lsm/rtree/impls/LSMRTree.java | 23 ++++++------
.../impls/LSMRTreeWithAntiMatterTuples.java | 11 +++---
.../am/lsm/rtree/utils/LSMRTreeUtils.java | 18 ++++-----
.../am/rtree/dataflow/RTreeDataflowHelper.java | 8 ++--
.../dataflow/RTreeDataflowHelperFactory.java | 6 ++-
.../storage/am/rtree/util/RTreeUtils.java | 3 +-
.../am/lsm/btree/LSMBTreeExamplesTest.java | 4 +-
...MBTreeModificationOperationCallbackTest.java | 3 +-
.../LSMBTreeSearchOperationCallbackTest.java | 3 +-
.../multithread/LSMBTreeMultiThreadTest.java | 6 +--
.../am/lsm/btree/perf/LSMTreeRunner.java | 6 +--
.../am/lsm/btree/util/LSMBTreeTestContext.java | 2 +-
.../util/LSMInvertedIndexTestContext.java | 4 +-
.../am/lsm/rtree/LSMRTreeExamplesTest.java | 2 +-
...SMRTreeWithAntiMatterTuplesExamplesTest.java | 2 +-
.../multithread/LSMRTreeMultiThreadTest.java | 6 +--
...TreeWithAntiMatterTuplesMultiThreadTest.java | 17 +++++----
.../am/lsm/rtree/util/LSMRTreeTestContext.java | 2 +-
...LSMRTreeWithAntiMatterTuplesTestContext.java | 10 +++--
.../storage/am/rtree/RTreeExamplesTest.java | 2 +-
.../rtree/multithread/RTreeMultiThreadTest.java | 6 +--
.../am/rtree/utils/RTreeTestContext.java | 2 +-
81 files changed, 379 insertions(+), 290 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 129201d..c71b352 100644
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -74,7 +74,7 @@ public interface IMetadataProvider<S, I> {
/**
* Creates the insert runtime of IndexInsertDeletePOperator, which models
* insert/delete operations into a secondary index.
- *
+ *
* @param dataSource
* Target secondary index.
* @param propagatedSchema
@@ -113,7 +113,7 @@ public interface IMetadataProvider<S, I> {
/**
* Creates the delete runtime of IndexInsertDeletePOperator, which models
* insert/delete operations into a secondary index.
- *
+ *
* @param dataSource
* Target secondary index.
* @param propagatedSchema
@@ -153,7 +153,7 @@ public interface IMetadataProvider<S, I> {
* Creates the TokenizeOperator for IndexInsertDeletePOperator, which tokenizes
* secondary key into [token, number of token] pair in a length-partitioned index.
* In case of non length-partitioned index, it tokenizes secondary key into [token].
- *
+ *
* @param dataSource
* Target secondary index.
* @param propagatedSchema
@@ -182,8 +182,8 @@ public interface IMetadataProvider<S, I> {
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
- ILogicalExpression filterExpr, RecordDescriptor recordDesc,
- JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException;
+ ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec,
+ boolean bulkload) throws AlgebricksException;
public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
index 0a7ffa2..8d1e370 100644
--- a/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
+++ b/hyracks/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileSplit.java
@@ -55,4 +55,9 @@ public class FileSplit implements Serializable {
public int getIODeviceId() {
return ioDeviceId;
}
+
+ @Override
+ public String toString() {
+ return file.toString();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index a18d496..499d63f 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -53,7 +53,7 @@ import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
// This example will insert tuples into the primary and secondary index using an insert pipeline
-public class InsertPipelineExample {
+public class InsertPipelineExample {
private static class Options {
@Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
public String host;
@@ -72,7 +72,7 @@ public class InsertPipelineExample {
@Option(name = "-secondary-btreename", usage = "B-Tree file name of secondary index", required = true)
public String secondaryBTreeName;
-
+
@Option(name = "-frame-size", usage = "Hyracks frame size (default: 32768)", required = false)
public int frameSize = 32768;
}
@@ -146,7 +146,7 @@ public class InsertPipelineExample {
// B-Tree tuple, etc.
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
- IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor primaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index 3ea0ad9..ea0b28f 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -149,7 +149,7 @@ public class PrimaryIndexBulkLoadExample {
// to field 0 of B-Tree tuple,
// etc.
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
- IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, recDesc,
storageManager, lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null,
fieldPermutation, 0.7f, false, 1000L, true, dataflowHelperFactory);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index 9029b2e..8b9eed3 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -139,7 +139,7 @@ public class PrimaryIndexSearchExample {
// into search op
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
- IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
lcManagerProvider, btreeSplitProvider, typeTraits, comparatorFactories, null, lowKeyFields,
highKeyFields, true, true, dataflowHelperFactory, false, false, null,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 62d0487..6ad0a63 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -120,7 +120,7 @@ public class SecondaryIndexBulkLoadExample {
// use a disk-order scan to read primary index
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
- IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec,
recDesc, storageManager, lcManagerProvider, primarySplitProvider, primaryTypeTraits,
dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 6d1f20e..6a218d6 100644
--- a/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -165,7 +165,7 @@ public class SecondaryIndexSearchExample {
IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
options.secondaryBTreeName);
- IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory(true);
BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
storageManager, lcManagerProvider, secondarySplitProvider, secondaryTypeTraits,
searchComparatorFactories, null, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeOperatorTestHelper.java
index 6d096be..9107d9d 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/btree/BTreeOperatorTestHelper.java
@@ -22,7 +22,7 @@ import edu.uci.ics.hyracks.tests.am.common.TreeOperatorTestHelper;
public class BTreeOperatorTestHelper extends TreeOperatorTestHelper {
public IIndexDataflowHelperFactory createDataFlowHelperFactory() {
- return new BTreeDataflowHelperFactory();
+ return new BTreeDataflowHelperFactory(true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
index 2aa037d..a77a71d 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/AbstractfWordInvertedIndexTest.java
@@ -96,7 +96,7 @@ public abstract class AbstractfWordInvertedIndexTest extends AbstractIntegration
DEFAULT_MEM_PAGE_SIZE, DEFAULT_MEM_NUM_PAGES);
protected IStorageManagerInterface storageManager = new TestStorageManagerInterface();
protected IIndexLifecycleManagerProvider lcManagerProvider = new TestIndexLifecycleManagerProvider();
- protected IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory();
+ protected IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory(true);
protected IIndexDataflowHelperFactory invertedIndexDataflowHelperFactory;
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
index 5d60f9d..e20b0ed 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/PartitionedWordInvertedIndexTest.java
@@ -55,7 +55,7 @@ public class PartitionedWordInvertedIndexTest extends AbstractfWordInvertedIndex
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null, null, null, null,
- null, null);
+ null, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
index 400a6b4..918611c 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/invertedindex/WordInvertedIndexTest.java
@@ -50,7 +50,7 @@ public class WordInvertedIndexTest extends AbstractfWordInvertedIndexTest {
new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
NoOpIOOperationCallback.INSTANCE, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null, null, null, null,
- null, null);
+ null, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
index 849f650..25cb47a 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/btree/LSMBTreeOperatorTestHelper.java
@@ -43,7 +43,7 @@ public class LSMBTreeOperatorTestHelper extends LSMTreeOperatorTestHelper {
return new LSMBTreeDataflowHelperFactory(virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE,
- DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, true, null, null, null, null);
+ DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, true, null, null, null, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index 0799f99..bc632f5 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -51,6 +51,6 @@ public class LSMRTreeOperatorTestHelper extends LSMTreeOperatorTestHelper {
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null,
- btreeFields, null, null, null);
+ btreeFields, null, null, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
index df8b062..f229356 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
@@ -50,6 +50,6 @@ public class LSMRTreeWithAntiMatterTuplesOperatorTestHelper extends LSMTreeOpera
btreeComparatorFactories, virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, null,
- null, null, null);
+ null, null, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
index 8a335b5..76ea772 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/AbstractRTreeOperatorTest.java
@@ -87,7 +87,7 @@ public abstract class AbstractRTreeOperatorTest extends AbstractIntegrationTest
protected final IStorageManagerInterface storageManager = new TestStorageManagerInterface();
protected final IIndexLifecycleManagerProvider lcManagerProvider = new TestIndexLifecycleManagerProvider();
protected IIndexDataflowHelperFactory rtreeDataflowHelperFactory;
- protected IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory();
+ protected IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory(true);
// field, type and key declarations for primary index
protected final int primaryFieldCount = 10;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java
index ce56c3d..a793a91 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeOperatorTestHelper.java
@@ -26,8 +26,8 @@ public class RTreeOperatorTestHelper extends TreeOperatorTestHelper {
public IIndexDataflowHelperFactory createDataFlowHelperFactory(
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
- IBinaryComparatorFactory[] btreeComparatorFactories) {
- return new RTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType);
+ IBinaryComparatorFactory[] btreeComparatorFactories, boolean durable) {
+ return new RTreeDataflowHelperFactory(valueProviderFactories, rtreePolicyType, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
index 7908d2b..18a8cfa 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexInsertOperatorTest.java
@@ -49,6 +49,7 @@ public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperator
this.rTreeType = RTreeType.RTREE;
}
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -127,7 +128,7 @@ public class RTreeSecondaryIndexInsertOperatorTest extends AbstractRTreeOperator
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
int[] btreeFields) throws TreeIndexException {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, null);
+ rtreePolicyType, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
index ea88e20..b18c053 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexScanOperatorTest.java
@@ -47,6 +47,7 @@ public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTe
this.rTreeType = RTreeType.RTREE;
}
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -109,7 +110,7 @@ public class RTreeSecondaryIndexScanOperatorTest extends AbstractRTreeOperatorTe
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
int[] btreeFields) {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, null);
+ rtreePolicyType, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index 432c2d7..f2239b9 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -48,6 +48,7 @@ public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperator
this.rTreeType = RTreeType.RTREE;
}
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -123,7 +124,7 @@ public class RTreeSecondaryIndexSearchOperatorTest extends AbstractRTreeOperator
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
int[] btreeFields) throws TreeIndexException {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, null);
+ rtreePolicyType, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
index 438e17d..5483e90 100644
--- a/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
+++ b/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/am/rtree/RTreeSecondaryIndexStatsOperatorTest.java
@@ -39,6 +39,7 @@ public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorT
this.rTreeType = RTreeType.RTREE;
}
+ @Override
@Before
public void setup() throws Exception {
super.setup();
@@ -72,7 +73,7 @@ public class RTreeSecondaryIndexStatsOperatorTest extends AbstractRTreeOperatorT
IBinaryComparatorFactory[] btreeComparatorFactories, ILinearizeComparatorFactory linearizerCmpFactory,
int[] btreeFields) {
return ((RTreeOperatorTestHelper) testHelper).createDataFlowHelperFactory(secondaryValueProviderFactories,
- rtreePolicyType, null);
+ rtreePolicyType, null, true);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
index 1e0125a..01376c1 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -26,8 +26,10 @@ import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
public class BTreeDataflowHelper extends TreeIndexDataflowHelper {
- public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
- super(opDesc, ctx, partition);
+
+ public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+ boolean durable) {
+ super(opDesc, ctx, partition, durable);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
index 2a17838..8652ed3 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
@@ -24,9 +24,15 @@ public class BTreeDataflowHelperFactory implements IIndexDataflowHelperFactory {
private static final long serialVersionUID = 1L;
+ private final boolean durable;
+
+ public BTreeDataflowHelperFactory(boolean durable) {
+ this.durable = durable;
+ }
+
@Override
public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition) {
- return new BTreeDataflowHelper(opDesc, ctx, partition);
+ return new BTreeDataflowHelper(opDesc, ctx, partition, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 5e07e0c..d67f585 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -40,10 +40,12 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
protected final FileReference file;
protected final int partition;
protected final int ioDeviceId;
+ protected final boolean durable;
protected IIndex index;
- public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx, int partition) {
+ public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx, int partition,
+ boolean durable) {
this.opDesc = opDesc;
this.ctx = ctx;
this.lcManager = opDesc.getLifecycleManagerProvider().getLifecycleManager(ctx);
@@ -53,14 +55,17 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
this.ioDeviceId = opDesc.getFileSplitProvider().getFileSplits()[partition].getIODeviceId();
this.file = new FileReference(new File(IndexFileNameUtil.prepareFileName(opDesc.getFileSplitProvider()
.getFileSplits()[partition].getLocalFile().getFile().getPath(), ioDeviceId)));
+ this.durable = durable;
}
protected abstract IIndex createIndexInstance() throws HyracksDataException;
+ @Override
public IIndex getIndexInstance() {
return index;
}
+ @Override
public void create() throws HyracksDataException {
synchronized (lcManager) {
long resourceID = getResourceID();
@@ -92,6 +97,7 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
}
}
+ @Override
public void open() throws HyracksDataException {
synchronized (lcManager) {
long resourceID = getResourceID();
@@ -109,12 +115,14 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
}
}
+ @Override
public void close() throws HyracksDataException {
synchronized (lcManager) {
lcManager.close(getResourceID());
}
}
+ @Override
public void destroy() throws HyracksDataException {
synchronized (lcManager) {
long resourceID = getResourceID();
@@ -132,10 +140,12 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
}
}
+ @Override
public FileReference getFileReference() {
return file;
}
+ @Override
public long getResourceID() throws HyracksDataException {
LocalResource localResource = localResourceRepository.getResourceByName(file.getFile().getPath());
if (localResource == null) {
@@ -145,6 +155,7 @@ public abstract class IndexDataflowHelper implements IIndexDataflowHelper {
}
}
+ @Override
public IHyracksTaskContext getTaskContext() {
return ctx;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
index de7f12b..fe3eed8 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
@@ -23,8 +23,9 @@ import edu.uci.ics.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
public abstract class TreeIndexDataflowHelper extends IndexDataflowHelper {
- public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) {
- super(opDesc, ctx, partition);
+ public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+ boolean durable) {
+ super(opDesc, ctx, partition, durable);
}
public ITreeIndexCursor createDiskOrderScanCursor(ITreeIndexFrame leafFrame) throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
index 983d893..b913e28 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelper.java
@@ -31,23 +31,25 @@ import edu.uci.ics.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
- private int version;
+ private final int version;
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version,
+ boolean durable) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, false, null, null, null, null);
+ ioOpCallbackFactory, false, null, null, null, null, durable);
this.version = version;
}
public ExternalBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck, int version,
+ boolean durable) {
this(opDesc, ctx, partition, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version);
+ ioScheduler, ioOpCallbackFactory, needKeyDupCheck, version, durable);
}
@Override
@@ -77,7 +79,7 @@ public class ExternalBTreeDataflowHelper extends LSMBTreeDataflowHelper {
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
- ioOpCallbackFactory.createIOOperationCallback(), getVersion());
+ ioOpCallbackFactory.createIOOperationCallback(), getVersion(), durable);
}
public int getVersion() {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
index d0e37f3..b9457d3 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeDataflowHelperFactory.java
@@ -29,14 +29,14 @@ public class ExternalBTreeDataflowHelperFactory extends AbstractLSMIndexDataflow
private static final long serialVersionUID = 1L;
- private int version;
+ private final int version;
public ExternalBTreeDataflowHelperFactory(ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- double bloomFilterFalsePositiveRate, int version) {
+ double bloomFilterFalsePositiveRate, int version, boolean durable) {
super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
- ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null);
+ ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null, durable);
this.version = version;
}
@@ -45,7 +45,7 @@ public class ExternalBTreeDataflowHelperFactory extends AbstractLSMIndexDataflow
int partition) {
return new ExternalBTreeDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, false, version, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
index 768dd05..ff19c54 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelper.java
@@ -28,15 +28,15 @@ import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.AbstractLSMIndexDatafl
public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDataflowHelper {
- private int[] buddyBtreeFields;
- private int version;
+ private final int[] buddyBtreeFields;
+ private final int version;
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- int[] buddyBtreeFields, int version) {
+ int[] buddyBtreeFields, int version, boolean durable) {
super(opDesc, ctx, partition, null, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, null,
- null, null);
+ null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
@@ -44,9 +44,10 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
public ExternalBTreeWithBuddyDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version) {
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] buddyBtreeFields, int version,
+ boolean durable) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, null, null, null);
+ ioOpCallbackFactory, null, null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
@@ -78,7 +79,7 @@ public class ExternalBTreeWithBuddyDataflowHelper extends AbstractLSMIndexDatafl
.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(), treeOpDesc
.getTreeIndexComparatorFactories(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory
.getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(),
- buddyBtreeFields, version);
+ buddyBtreeFields, version, durable);
}
public int getTargetVersion() {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
index cda5fff..dc1939a 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyDataflowHelperFactory.java
@@ -29,14 +29,14 @@ public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMInde
private static final long serialVersionUID = 1L;
private final int[] buddyBtreeFields;
- private int version;
+ private final int version;
public ExternalBTreeWithBuddyDataflowHelperFactory(ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- double bloomFilterFalsePositiveRate, int[] buddyBtreeFields, int version) {
+ double bloomFilterFalsePositiveRate, int[] buddyBtreeFields, int version, boolean durable) {
super(null, mergePolicyFactory, mergePolicyProperties, opTrackerFactory, ioSchedulerProvider,
- ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null);
+ ioOpCallbackFactory, bloomFilterFalsePositiveRate, null, null, null, durable);
this.buddyBtreeFields = buddyBtreeFields;
this.version = version;
}
@@ -46,7 +46,7 @@ public class ExternalBTreeWithBuddyDataflowHelperFactory extends AbstractLSMInde
int partition) {
return new ExternalBTreeWithBuddyDataflowHelper(opDesc, ctx, partition, bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
- ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version);
+ ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, buddyBtreeFields, version, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
index b4bd166..2629a1d 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelper.java
@@ -42,10 +42,10 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, boolean needKeyDupCheck,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
- int[] filterFields) {
+ int[] filterFields, boolean durable) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy,
opTrackerFactory, ioScheduler, ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields);
+ filterCmpFactories, btreeFields, filterFields, durable);
}
public LSMBTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
@@ -53,9 +53,9 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields) {
+ int[] btreeFields, int[] filterFields, boolean durable) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
- ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields);
+ ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
}
@@ -68,6 +68,6 @@ public class LSMBTreeDataflowHelper extends AbstractLSMIndexDataflowHelper {
treeOpDesc.getTreeIndexComparatorFactories(), treeOpDesc.getTreeIndexBloomFilterKeyFields(),
bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler,
ioOpCallbackFactory.createIOOperationCallback(), needKeyDupCheck, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields);
+ btreeFields, filterFields, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
index f59392c..b4515da 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeDataflowHelperFactory.java
@@ -40,10 +40,10 @@ public class LSMBTreeDataflowHelperFactory extends AbstractLSMIndexDataflowHelpe
ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationSchedulerProvider ioSchedulerProvider,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, double bloomFilterFalsePositiveRate,
boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories,
- int[] btreeFields, int[] filterFields) {
+ int[] btreeFields, int[] filterFields, boolean durable) {
super(virtualBufferCacheProvider, mergePolicyFactory, mergePolicyProperties, opTrackerFactory,
ioSchedulerProvider, ioOpCallbackFactory, bloomFilterFalsePositiveRate, filterTypeTraits,
- filterCmpFactories, filterFields);
+ filterCmpFactories, filterFields, durable);
this.needKeyDupCheck = needKeyDupCheck;
this.btreeFields = btreeFields;
}
@@ -55,6 +55,6 @@ public class LSMBTreeDataflowHelperFactory extends AbstractLSMIndexDataflowHelpe
virtualBufferCacheProvider.getVirtualBufferCaches(ctx), bloomFilterFalsePositiveRate,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory,
ioSchedulerProvider.getIOScheduler(ctx), ioOpCallbackFactory, needKeyDupCheck, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields);
+ filterCmpFactories, btreeFields, filterFields, durable);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index b417152..088a15b 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -44,7 +44,6 @@ import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
@@ -54,10 +53,11 @@ import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
@@ -76,7 +76,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
private final LSMBTreeDiskComponentFactory transactionComponentFactory;
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
- private List<ILSMComponent> secondDiskComponents;
+ private final List<ILSMComponent> secondDiskComponents;
// A pointer that points to the current most recent list (either
// diskComponents = 0, or secondDiskComponents = 1). It starts with -1 to
// indicate first time activation
@@ -90,11 +90,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version) {
+ ILSMIOOperationCallback ioOpCallback, TreeIndexFactory<BTree> transactionBTreeFactory, int version,
+ boolean durable) {
super(interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider,
- fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false);
- this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
+ fieldCount, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
+ this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory,
+ bloomFilterFactory, null);
this.secondDiskComponents = new LinkedList<ILSMComponent>();
this.interiorFrameFactory = interiorFrameFactory;
this.version = version;
@@ -185,9 +187,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
}
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
- BTree firstBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
- BTree lastBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1))
- .getBTree();
+ BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
+ BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
@@ -217,6 +218,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
}
// This function is used when a new component is to be committed.
+ @Override
public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
// determine which list is the new one and flip the pointer
List<ILSMComponent> newerList;
@@ -381,12 +383,12 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
@Override
public void validate() throws HyracksDataException {
for (ILSMComponent c : diskComponents) {
- BTree btree = (BTree) ((LSMBTreeDiskComponent) c).getBTree();
+ BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
btree.validate();
}
for (ILSMComponent c : secondDiskComponents) {
if (!diskComponents.contains(c)) {
- BTree btree = (BTree) ((LSMBTreeDiskComponent) c).getBTree();
+ BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
btree.validate();
}
}
@@ -474,8 +476,8 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
private boolean endedBloomFilterLoad = false;
- private boolean isTransaction;
- private ITreeIndexTupleWriterFactory frameTupleWriterFactory;
+ private final boolean isTransaction;
+ private final ITreeIndexTupleWriterFactory frameTupleWriterFactory;
public LSMTwoPCBTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, boolean isTransaction) throws TreeIndexException, HyracksDataException {
@@ -595,9 +597,9 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
@Override
public void abort() {
- try{
+ try {
cleanupArtifacts();
- } catch(Exception e){
+ } catch (Exception e) {
// Do nothing
}
}
@@ -652,14 +654,17 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
return insertLeafFrameFactory;
}
+ @Override
public int getCurrentVersion() {
return version;
}
+ @Override
public List<ILSMComponent> getFirstComponentList() {
return diskComponents;
}
+ @Override
public List<ILSMComponent> getSecondComponentList() {
return secondDiskComponents;
}
@@ -692,9 +697,9 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
throw new TreeIndexException(e);
}
}
-
+
@Override
- public boolean hasMemoryComponents(){
+ public boolean hasMemoryComponents() {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 11c319f..1a4b85f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -27,8 +27,8 @@ import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoader;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexCursor;
@@ -45,7 +45,6 @@ import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOperation;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponent;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMComponentFactory;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMHarness;
@@ -57,11 +56,12 @@ import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ITwoPCIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
-import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.ExternalIndexHarness;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMOperationType;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
@@ -76,7 +76,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
private final IBinaryComparatorFactory[] btreeCmpFactories;
private final IBinaryComparatorFactory[] buddyBtreeCmpFactories;
- private int[] buddyBTreeFields;
+ private final int[] buddyBTreeFields;
// Common for in-memory and on-disk components.
private final ITreeIndexFrameFactory btreeInteriorFrameFactory;
@@ -85,7 +85,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// A second disk component list that will be used when a transaction is
// committed and will be seen by subsequent accessors
- private List<ILSMComponent> secondDiskComponents;
+ private final List<ILSMComponent> secondDiskComponents;
private int version = -1;
public ExternalBTreeWithBuddy(ITreeIndexFrameFactory btreeInteriorFrameFactory,
@@ -96,9 +96,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
- int[] buddyBTreeFields, int version) {
+ int[] buddyBTreeFields, int version, boolean durable) {
super(diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
- ioScheduler, ioOpCallback);
+ ioScheduler, ioOpCallback, durable);
this.btreeCmpFactories = btreeCmpFactories;
this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
this.buddyBTreeFields = buddyBTreeFields;
@@ -293,14 +293,14 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws TreeIndexException {
try {
- return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex,
- false);
+ return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
} catch (HyracksDataException e) {
throw new TreeIndexException(e);
}
}
// For transaction bulk load <- could consolidate with the above method ->
+ @Override
public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex) throws TreeIndexException {
try {
@@ -375,10 +375,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
.get(secondDiskComponents.size() - 1);
}
- ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation((ILSMIndexAccessorInternal) accessor,
- mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs
- .getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback,
- fileManager.getBaseDir(), keepDeleteTuples));
+ ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor,
+ relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
+ relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir(), keepDeleteTuples));
}
// This method creates the appropriate opContext for the targeted version
@@ -509,6 +508,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
// This function is used when a new component is to be committed -- is
// called by the harness.
+ @Override
public void commitTransactionDiskComponent(ILSMComponent newComponent) throws HyracksDataException {
// determine which list is the new one and flip the pointer
@@ -663,7 +663,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
private boolean cleanedUpArtifacts = false;
private boolean isEmptyComponent = true;
private boolean endedBloomFilterLoad = false;
- private boolean isTransaction;
+ private final boolean isTransaction;
public LSMTwoPCBTreeWithBuddyBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, boolean isTransaction) throws TreeIndexException, HyracksDataException {
@@ -824,14 +824,17 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
}
}
+ @Override
public int getCurrentVersion() {
return version;
}
+ @Override
public List<ILSMComponent> getFirstComponentList() {
return diskComponents;
}
+ @Override
public List<ILSMComponent> getSecondComponentList() {
return secondDiskComponents;
}
@@ -865,9 +868,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
throw new TreeIndexException(e);
}
}
-
+
@Override
- public boolean hasMemoryComponents(){
+ public boolean hasMemoryComponents() {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7f7c5730/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 2899eff..6d2f78c 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -24,7 +24,6 @@ import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomCalculations;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilter;
import edu.uci.ics.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
@@ -103,10 +102,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
LSMComponentFilterManager filterManager, double bloomFilterFalsePositiveRate,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields) {
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
+ boolean durable) {
super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
- filterManager, filterFields);
+ filterManager, filterFields, durable);
int i = 0;
for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
@@ -136,9 +136,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
- ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck) {
+ ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
super(diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate,
- mergePolicy, opTracker, ioScheduler, ioOpCallback);
+ mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
this.insertLeafFrameFactory = insertLeafFrameFactory;
this.deleteLeafFrameFactory = deleteLeafFrameFactory;
this.cmpFactories = cmpFactories;
@@ -513,9 +513,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
returnDeletedTuples = true;
}
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
- BTree firstBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
- BTree lastBTree = (BTree) ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1))
- .getBTree();
+ BTree firstBTree = ((LSMBTreeDiskComponent) mergingComponents.get(0)).getBTree();
+ BTree lastBTree = ((LSMBTreeDiskComponent) mergingComponents.get(mergingComponents.size() - 1)).getBTree();
FileReference firstFile = diskFileMapProvider.lookupFileName(firstBTree.getFileId());
FileReference lastFile = diskFileMapProvider.lookupFileName(lastBTree.getFileId());
LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFile.getFile()
@@ -766,6 +765,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
return diskBufferCache;
}
+ @Override
public IBinaryComparatorFactory[] getComparatorFactories() {
return cmpFactories;
}
@@ -831,7 +831,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
}
List<ILSMComponent> immutableComponents = diskComponents;
for (ILSMComponent c : immutableComponents) {
- BTree btree = (BTree) ((LSMBTreeDiskComponent) c).getBTree();
+ BTree btree = ((LSMBTreeDiskComponent) c).getBTree();
btree.validate();
}
}