You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/04/30 20:50:43 UTC

[8/8] asterixdb git commit: [NO ISSUE][STO] Add consistency to flush lifecycle

[NO ISSUE][STO] Add consistency to flush lifecycle

- user model changes: no
- storage format changes: yes
  - renamed AbstractLSMIOOperationCallbackFactory
    to LSMIOOperationCallbackFactory
  - useless classes have been removed.
    - LSMBTreeIOOperationCallbackFactory
    - LSMBTreeWithBuddyIOOperationCallbackFactory
    - LSMInvertedIndexIOOperationCallbackFactory
    - LSMRTreeIOOperationCallbackFactory
- interface changes: yes

Details:
- Previously, flushes have different lifecycle depending
  on the memory component state
  - not allocated
  - allocated
  - modified
- In certain cases, flush operations are skipped alltogether
- IO Operation callbacks became complicated and difficult
  to maintain since calls are done differently in different
  cases.
- In certain cases, afterFinalize is called on the IO
  Operation callbacks even if beforeOperation was never
  called.
- In this change, flushes go through the same lifecycle
  events regardless of the state of the memory component.
- In addition, primary and secondary memory components
  would reside in different virtual buffer caches due
  to skipped flushes, or due to having the secondary
  index created when the primary index's memory component
  is residing on the virtual buffer cache with index !=0.
- Moreover, when flushes are lagging and all memory
  components are being flushed, search operations assumes
  the oldest of the memory component is the newest and
  produces incorrect results.
- In addition, in case of a failed flush of a component,
  the IO scheduler would skip it and flush the next
  component. This would produce a bad state on disk.
- In this change, a failed flush can be retried. otherwise,
  all future flushes of the component fail due to the failure
  of the previously failed flush.
- Previously, when a component fails to modify an index due
  to flush failures, it assumes disk is full.
- With this change, the modification failure reports the
  original cause of the failed flush.

Change-Id: I29f7992ec6c0f71c5b63d45800b2fb590d651e4b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2584
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Murtadha Hubail <mh...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/4ed0eac5
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/4ed0eac5
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/4ed0eac5

Branch: refs/heads/master
Commit: 4ed0eac5cb4cf5b849bc65847bd4e979d01a210f
Parents: 9410ddd
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Sat Apr 28 18:26:08 2018 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Sun Apr 29 09:48:55 2018 -0700

----------------------------------------------------------------------
 .../org/apache/asterix/app/nc/HaltCallback.java |  49 ++
 .../asterix/app/nc/NCAppRuntimeContext.java     |  11 +-
 .../apache/asterix/app/nc/RecoveryManager.java  |  27 +-
 .../test/dataflow/ComponentRollbackTest.java    |  82 +++-
 .../test/dataflow/IoCallbackFailureTest.java    |   2 +-
 .../test/dataflow/LSMFlushRecoveryTest.java     |  46 +-
 .../dataflow/MultiPartitionLSMIndexTest.java    |  10 +-
 .../SearchCursorComponentSwitchTest.java        |  12 +-
 .../asterix/test/dataflow/StorageTestUtils.java |   6 +-
 .../asterix/test/dataflow/TestDataset.java      |   2 +-
 .../TestLsmBtreeIoOpCallbackFactory.java        | 170 -------
 .../dataflow/TestLsmIoOpCallbackFactory.java    | 192 ++++++++
 .../TestPrimaryIndexOperationTracker.java       |   2 +-
 .../test/dataflow/TransactionAbortTest.java     |   2 +-
 .../asterix/test/metadata/MetadataTxnTest.java  |   8 +-
 .../asterix/test/storage/DiskIsFullTest.java    | 182 -------
 .../common/api/IDatasetInfoProvider.java        |  36 ++
 .../api/ILSMComponentIdGeneratorFactory.java    |  40 ++
 .../common/context/BaseOperationTracker.java    |   7 +-
 .../context/CorrelatedPrefixMergePolicy.java    |   2 +-
 .../common/context/DatasetInfoProvider.java     |  42 ++
 .../DatasetLSMComponentIdGeneratorFactory.java  |   3 +-
 .../common/context/DatasetLifecycleManager.java |  46 +-
 .../context/PrimaryIndexOperationTracker.java   |  56 ++-
 .../asterix/common/dataflow/LSMIndexUtil.java   |  12 +-
 .../common/exceptions/ExceptionUtils.java       |  21 +-
 .../AbstractLSMIOOperationCallback.java         | 287 -----------
 ...tractLSMIndexIOOperationCallbackFactory.java |  86 ----
 .../LSMBTreeIOOperationCallback.java            |  32 --
 .../LSMBTreeIOOperationCallbackFactory.java     |  39 --
 .../LSMBTreeWithBuddyIOOperationCallback.java   |  31 --
 ...TreeWithBuddyIOOperationCallbackFactory.java |  39 --
 .../ioopcallbacks/LSMIOOperationCallback.java   | 227 +++++++++
 .../LSMIndexIOOperationCallbackFactory.java     |  76 +++
 .../LSMInvertedIndexIOOperationCallback.java    |  32 --
 ...InvertedIndexIOOperationCallbackFactory.java |  40 --
 .../LSMRTreeIOOperationCallback.java            |  32 --
 .../LSMRTreeIOOperationCallbackFactory.java     |  39 --
 .../asterix/common/utils/StorageConstants.java  |   2 +-
 .../CorrelatedPrefixMergePolicyTest.java        |   6 +-
 .../AbstractLSMIOOperationCallbackTest.java     | 315 -------------
 .../LSMBTreeIOOperationCallbackTest.java        |  39 --
 ...SMBTreeWithBuddyIOOperationCallbackTest.java |  39 --
 .../LSMIOOperationCallbackTest.java             | 217 +++++++++
 ...LSMInvertedIndexIOOperationCallbackTest.java |  39 --
 .../LSMRTreeIOOperationCallbackTest.java        |  39 --
 .../test/ioopcallbacks/TestFlushOperation.java  |  56 +++
 .../ioopcallbacks/TestLSMIndexAccessor.java     | 183 ++++++++
 .../TestLSMIndexOperationContext.java           |  25 +-
 .../metadata/api/IActiveEntityController.java   |   2 +-
 .../metadata/bootstrap/MetadataBootstrap.java   |  11 +-
 .../asterix/metadata/entities/Dataset.java      |  33 +-
 .../replication/sync/IndexSynchronizer.java     |   6 +-
 ...irstRuleCheckFixpointRuleControllerTest.java |   2 +
 .../hyracks/api/exceptions/ErrorCode.java       |   2 +-
 .../apache/hyracks/api/util/ExceptionUtils.java |  11 +
 .../api/util/SingleThreadEventProcessor.java    |  10 +-
 .../src/main/resources/errormsg/en.properties   |   2 +-
 .../am/common/ophelpers/IndexOperation.java     |   3 +-
 .../am/lsm/btree/impls/ExternalBTree.java       |  26 +-
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  32 +-
 .../storage/am/lsm/btree/impls/LSMBTree.java    |  17 +-
 .../btree/impls/LSMBTreeMemoryComponent.java    |   4 +-
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |   2 -
 .../btree/impls/LSMBTreeRangeSearchCursor.java  |   5 +-
 .../impls/LSMBTreeWithBuddyMemoryComponent.java |   6 +-
 .../am/lsm/btree/utils/LSMBTreeUtil.java        |   5 -
 ...AbstractLSMWithBloomFilterDiskComponent.java |   5 +-
 .../AbstractLSMWithBuddyMemoryComponent.java    |  63 ---
 .../common/api/IIoOperationFailedCallback.java  |  44 ++
 .../am/lsm/common/api/ILSMComponent.java        |   9 +
 .../common/api/ILSMComponentIdGenerator.java    |   9 +-
 .../api/ILSMComponentIdGeneratorFactory.java    |  31 --
 .../am/lsm/common/api/ILSMDiskComponent.java    |   5 +-
 .../common/api/ILSMDiskComponentBulkLoader.java |   5 +
 .../storage/am/lsm/common/api/ILSMHarness.java  |  12 +-
 .../am/lsm/common/api/ILSMIOOperation.java      |  65 ++-
 .../lsm/common/api/ILSMIOOperationCallback.java |  33 +-
 .../api/ILSMIOOperationCallbackFactory.java     |  13 +
 .../storage/am/lsm/common/api/ILSMIndex.java    |  36 +-
 .../am/lsm/common/api/ILSMIndexAccessor.java    |  24 +-
 .../common/api/ILSMIndexOperationContext.java   |  25 +-
 .../am/lsm/common/api/ILSMMemoryComponent.java  |  21 +-
 .../LSMIndexCompactOperatorNodePushable.java    |   2 +-
 .../lsm/common/impls/AbstractIoOperation.java   |  53 +++
 .../common/impls/AbstractLSMDiskComponent.java  |  41 +-
 .../am/lsm/common/impls/AbstractLSMIndex.java   | 297 ++++++++----
 .../impls/AbstractLSMIndexOperationContext.java |  24 +-
 .../impls/AbstractLSMMemoryComponent.java       | 118 +++--
 .../AbstractLSMWithBuddyMemoryComponent.java    |  63 +++
 .../lsm/common/impls/AsynchronousScheduler.java | 126 +++--
 .../BlockingIOOperationCallbackWrapper.java     |  69 ---
 .../ChainedLSMDiskComponentBulkLoader.java      |  15 +-
 .../impls/ComponentReplacementContext.java      |  12 +-
 .../lsm/common/impls/ConstantMergePolicy.java   |   6 +-
 .../am/lsm/common/impls/EmptyComponent.java     |   8 +-
 .../lsm/common/impls/ExternalIndexHarness.java  |  78 +--
 .../am/lsm/common/impls/FlushOperation.java     |   4 +-
 .../lsm/common/impls/IoOperationExecutor.java   | 121 +++++
 .../common/impls/LSMComponentIdGenerator.java   |  13 +-
 .../impls/LSMComponentIdGeneratorFactory.java   |  40 --
 .../storage/am/lsm/common/impls/LSMHarness.java | 354 ++++++--------
 .../impls/LSMIndexDiskComponentBulkLoader.java  |  48 +-
 .../lsm/common/impls/LSMTreeIndexAccessor.java  |  18 +-
 .../am/lsm/common/impls/LoadOperation.java      |  49 ++
 .../am/lsm/common/impls/MergeOperation.java     |   4 +-
 .../impls/MultitenantVirtualBufferCache.java    |   4 +
 .../impls/NoOpIOOperationCallbackFactory.java   |  36 +-
 .../am/lsm/common/impls/NoOpIoOperation.java    | 116 +++++
 .../impls/NoOpIoOperationFailedCallback.java    |  44 ++
 .../am/lsm/common/impls/PrefixMergePolicy.java  |   7 +-
 .../common/impls/StubIOOperationCallback.java   |  70 ---
 .../lsm/common/impls/SynchronousScheduler.java  |  77 ++-
 .../impls/SynchronousSchedulerProvider.java     |  10 +-
 .../lsm/common/impls/ThreadCountingTracker.java |   2 +-
 .../am/lsm/common/impls/TracedIOOperation.java  |  43 +-
 .../am/lsm/common/impls/VirtualBufferCache.java |   6 +
 .../am/lsm/common/util/IOOperationUtils.java    |  43 --
 .../invertedindex/impls/LSMInvertedIndex.java   |  20 +-
 .../impls/LSMInvertedIndexAccessor.java         |  18 +-
 .../impls/LSMInvertedIndexMemoryComponent.java  |   6 +-
 .../am/lsm/rtree/impls/AbstractLSMRTree.java    |   5 +-
 .../am/lsm/rtree/impls/ExternalRTree.java       |  27 +-
 .../storage/am/lsm/rtree/impls/LSMRTree.java    |  17 +-
 .../rtree/impls/LSMRTreeMemoryComponent.java    |   6 +-
 .../impls/LSMRTreeWithAntiMatterTuples.java     |  10 +-
 .../storage/common/IIndexBulkLoader.java        |   2 +-
 .../storage/common/buffercache/BufferCache.java |  19 +
 .../common/buffercache/DebugBufferCache.java    |   6 +
 .../common/buffercache/IBufferCache.java        |   9 +
 .../am/lsm/btree/EncapsulatingIoCallback.java   | 107 +++++
 .../btree/EncapsulatingIoCallbackFactory.java   |  71 +++
 .../btree/LSMBTreeComponentLifecycleTest.java   | 470 +++++++++++++++++++
 .../am/lsm/btree/LSMBTreeFileManagerTest.java   |   5 +-
 .../btree/LSMBTreeFilterMergeTestDriver.java    |  18 +-
 .../am/lsm/btree/LSMBTreeMergeFailTest.java     |  39 +-
 .../am/lsm/btree/LSMBTreeMergeTestDriver.java   |   3 +-
 ...MBTreeModificationOperationCallbackTest.java |  21 +-
 .../btree/LSMBTreeScanDiskComponentsTest.java   |   9 +-
 ...TreeUpdateInPlaceScanDiskComponentsTest.java |   5 +-
 .../am/lsm/btree/LSMBTreeUpdateInPlaceTest.java |  16 +-
 .../am/lsm/btree/impl/AllowTestOpCallback.java  |   2 +-
 .../btree/impl/CountingIoOperationCallback.java | 101 ++++
 .../CountingIoOperationCallbackFactory.java     |  48 ++
 .../am/lsm/btree/impl/ITestOpCallback.java      |   2 +-
 .../am/lsm/btree/impl/NoOpTestCallback.java     |  45 ++
 .../storage/am/lsm/btree/impl/TestLsmBtree.java |  87 +++-
 .../lsm/btree/impl/TestVirtualBufferCache.java  |   6 +
 .../btree/multithread/LSMBTreeTestWorker.java   |   2 +-
 .../am/lsm/btree/perf/LSMTreeRunner.java        |  19 +-
 .../am/lsm/btree/util/LSMBTreeTestHarness.java  |   8 +-
 .../lsm/common/test/PrefixMergePolicyTest.java  |   6 +-
 .../LSMInvertedIndexMergeTest.java              |   3 +-
 .../PartitionedLSMInvertedIndexMergeTest.java   |   3 +-
 .../common/LSMInvertedIndexTestHarness.java     |   4 +-
 .../multithread/LSMInvertedIndexTestWorker.java |   2 +-
 .../am/lsm/rtree/LSMRTreeMergeTestDriver.java   |   3 +-
 .../LSMRTreeWithAntiMatterTuplesDeleteTest.java |   4 +-
 .../rtree/multithread/LSMRTreeTestWorker.java   |   2 +-
 .../LSMRTreeWithAntiMatterTuplesTestWorker.java |   3 +-
 .../am/lsm/rtree/util/LSMRTreeTestHarness.java  |   4 +-
 .../java/org/apache/hyracks/util/ExitUtil.java  |   5 +-
 162 files changed, 3954 insertions(+), 2902 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
new file mode 100644
index 0000000..db34131
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc;
+
+import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class HaltCallback implements IIoOperationFailedCallback {
+    private static final Logger LOGGER = LogManager.getLogger();
+    public static final HaltCallback INSTANCE = new HaltCallback();
+
+    private HaltCallback() {
+    }
+
+    @Override
+    public void schedulerFailed(ILSMIOOperationScheduler scheduler, Throwable failure) {
+        LOGGER.error("IO Scheduler has failed", failure);
+        ExitUtil.halt(ExitUtil.EC_IO_SCHEDULER_FAILED);
+    }
+
+    @Override
+    public void operationFailed(ILSMIOOperation operation, Throwable t) {
+        LOGGER.error("Operation {} has failed", t);
+        if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+            ExitUtil.halt(ExitUtil.EC_FLUSH_FAILED);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 288e5f2..c3b1bae 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -52,7 +52,6 @@ import org.apache.asterix.common.config.TransactionProperties;
 import org.apache.asterix.common.context.DatasetLifecycleManager;
 import org.apache.asterix.common.context.DatasetMemoryManager;
 import org.apache.asterix.common.context.IStorageComponentProvider;
-import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.library.ILibraryManager;
 import org.apache.asterix.common.replication.IReplicationChannel;
@@ -169,7 +168,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
     }
 
     @Override
-    public void initialize(boolean initialRun) throws IOException, ACIDException {
+    public void initialize(boolean initialRun) throws IOException {
         ioManager = getServiceContext().getIoManager();
         threadExecutor =
                 MaintainedThreadNameExecutorService.newCachedThreadPool(getServiceContext().getThreadFactory());
@@ -177,19 +176,13 @@ public class NCAppRuntimeContext implements INcApplicationContext {
         IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
         IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
                 storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
-
-        AsynchronousScheduler.INSTANCE.init(getServiceContext().getThreadFactory());
-        lsmIOScheduler = AsynchronousScheduler.INSTANCE;
-
+        lsmIOScheduler = new AsynchronousScheduler(getServiceContext().getThreadFactory(), HaltCallback.INSTANCE);
         metadataMergePolicyFactory = new PrefixMergePolicyFactory();
         indexCheckpointManagerProvider = new IndexCheckpointManagerProvider(ioManager);
-
         ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
                 new PersistentLocalResourceRepositoryFactory(ioManager, indexCheckpointManagerProvider);
-
         localResourceRepository =
                 (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
-
         txnSubsystem = new TransactionSubsystem(this);
         IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
         SystemState systemState = recoveryMgr.getSystemState();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index d4e652d..efd173f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -48,7 +48,7 @@ import org.apache.asterix.common.context.DatasetInfo;
 import org.apache.asterix.common.context.IndexInfo;
 import org.apache.asterix.common.dataflow.DatasetLocalResource;
 import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
 import org.apache.asterix.common.storage.DatasetResourceReference;
 import org.apache.asterix.common.storage.IIndexCheckpointManager;
 import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
@@ -401,13 +401,12 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
                                 if (iInfo.isOpen() && iInfo.getPartition() == partition) {
                                     maxDiskLastLsn = resourceId2MaxLSNMap.get(iInfo.getResourceId());
                                     index = iInfo.getIndex();
-                                    AbstractLSMIOOperationCallback ioCallback =
-                                            (AbstractLSMIOOperationCallback) index.getIOOperationCallback();
+                                    LSMIOOperationCallback ioCallback =
+                                            (LSMIOOperationCallback) index.getIOOperationCallback();
                                     if (logRecord.getLSN() > maxDiskLastLsn
                                             && !index.isCurrentMutableComponentEmpty()) {
                                         // schedule flush
-                                        ioCallback.updateLastLSN(logRecord.getLSN());
-                                        redoFlush(index, logRecord);
+                                        redoFlush(index, logRecord, idGenerator.getId());
                                         redoCount++;
                                     } else {
                                         if (index.isMemoryComponentsAllocated()) {
@@ -417,7 +416,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
                                         } else {
                                             // otherwise, we refresh the id stored in ioCallback
                                             // to ensure the memory component receives correct Id upon activation
-                                            ioCallback.forceRefreshNextId();
+                                            ioCallback.forceRefreshNextId(idGenerator.getId());
                                         }
                                     }
                                 }
@@ -473,10 +472,10 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         long minFirstLSN = logMgr.getAppendLSN();
         if (!openIndexList.isEmpty()) {
             for (IIndex index : openIndexList) {
-                AbstractLSMIOOperationCallback ioCallback =
-                        (AbstractLSMIOOperationCallback) ((ILSMIndex) index).getIOOperationCallback();
+                LSMIOOperationCallback ioCallback =
+                        (LSMIOOperationCallback) ((ILSMIndex) index).getIOOperationCallback();
                 if (!((AbstractLSMIndex) index).isCurrentMutableComponentEmpty() || ioCallback.hasPendingFlush()) {
-                    firstLSN = ioCallback.getFirstLSN();
+                    firstLSN = ioCallback.getPersistenceLsn();
                     minFirstLSN = Math.min(minFirstLSN, firstLSN);
                 }
             }
@@ -823,11 +822,17 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         }
     }
 
-    private static void redoFlush(ILSMIndex index, ILogRecord logRecord) throws HyracksDataException {
+    private static void redoFlush(ILSMIndex index, ILogRecord logRecord, ILSMComponentId nextId)
+            throws HyracksDataException {
+        long flushLsn = logRecord.getLSN();
+        Map<String, Object> flushMap = new HashMap<>();
+        flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
         ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+        accessor.getOpContext().setParameters(flushMap);
         long minId = logRecord.getFlushingComponentMinId();
         long maxId = logRecord.getFlushingComponentMaxId();
         ILSMComponentId id = new LSMComponentId(minId, maxId);
+        flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, nextId);
         if (!index.getDiskComponents().isEmpty()) {
             ILSMDiskComponent diskComponent = index.getDiskComponents().get(0);
             ILSMComponentId maxDiskComponentId = diskComponent.getId();
@@ -837,7 +842,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
             }
         }
         index.getCurrentMemoryComponent().resetId(id, true);
-        accessor.scheduleFlush(index.getIOOperationCallback());
+        accessor.scheduleFlush();
     }
 
     private class JobEntityCommits {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
index 9ef531e..a33bda1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
@@ -20,7 +20,9 @@ package org.apache.asterix.test.dataflow;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Predicate;
 
 import org.apache.asterix.app.bootstrap.TestNodeController;
@@ -29,7 +31,7 @@ import org.apache.asterix.app.data.gen.TupleGenerator;
 import org.apache.asterix.app.nc.NCAppRuntimeContext;
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
-import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
 import org.apache.asterix.common.transactions.ITransactionContext;
 import org.apache.asterix.common.transactions.ITransactionManager;
 import org.apache.asterix.common.transactions.TransactionOptions;
@@ -50,6 +52,7 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
 import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
@@ -146,16 +149,27 @@ public class ComponentRollbackTest {
             StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
             ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            ILSMComponentId next =
+                    dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            Map<String, Object> flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             // rollback a memory component
             lsmAccessor.deleteComponents(memoryComponentsPredicate);
             StorageTestUtils.searchAndAssertCount(nc, PARTITION,
                     StorageTestUtils.TOTAL_NUM_OF_RECORDS - StorageTestUtils.RECORDS_PER_COMPONENT);
             // rollback the last disk component
             lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
             lsmAccessor.deleteComponents(pred);
             StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -203,7 +217,13 @@ public class ComponentRollbackTest {
             StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
             ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            ILSMComponentId next =
+                    dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            Map<String, Object> flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             // rollback a memory component
             lsmAccessor.deleteComponents(memoryComponentsPredicate);
             StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -227,9 +247,14 @@ public class ComponentRollbackTest {
             StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
             // rollback the last disk component
             lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
             lsmAccessor.deleteComponents(pred);
             StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -278,7 +303,13 @@ public class ComponentRollbackTest {
             // now that we enetered, we will rollback
             ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            ILSMComponentId next =
+                    dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            Map<String, Object> flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             // rollback a memory component
             lsmAccessor.deleteComponents(
                     c -> (c instanceof ILSMMemoryComponent && ((ILSMMemoryComponent) c).isModified()));
@@ -297,10 +328,14 @@ public class ComponentRollbackTest {
             // wait till firstSearcher enter the components
             secondSearcher.waitUntilEntered();
             lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
-
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-            ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+            next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+            flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+            flushMap = new HashMap<>();
+            flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+            flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+            lsmAccessor.getOpContext().setParameters(flushMap);
             DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
             lsmAccessor.deleteComponents(pred);
             // now that the rollback has completed, we will unblock the search
@@ -407,11 +442,11 @@ public class ComponentRollbackTest {
             // select the components to merge... the last three
             int numMergedComponents = 3;
             List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             for (int i = 0; i < numMergedComponents; i++) {
                 mergedComponents.add(diskComponents.get(i));
             }
-            mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+            mergeAccessor.scheduleMerge(mergedComponents);
             merger.waitUntilCount(1);
             // now that we enetered, we will rollback
             Rollerback rollerback = new Rollerback(lsmBtree, new DiskComponentLsnPredicate(lsn));
@@ -590,11 +625,11 @@ public class ComponentRollbackTest {
             // select the components to merge... the last three
             int numMergedComponents = 3;
             List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             for (int i = 0; i < numMergedComponents; i++) {
                 mergedComponents.add(diskComponents.get(i));
             }
-            mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+            mergeAccessor.scheduleMerge(mergedComponents);
             merger.waitUntilCount(1);
             // we will block search
             lsmBtree.clearSearchCallbacks();
@@ -659,12 +694,12 @@ public class ComponentRollbackTest {
             ILSMIndexAccessor mergeAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             // select the components to merge... the last three
             List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
-            long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+            long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
             int numMergedComponents = 3;
             for (int i = 0; i < numMergedComponents; i++) {
                 mergedComponents.add(diskComponents.get(i));
             }
-            mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+            mergeAccessor.scheduleMerge(mergedComponents);
             merger.waitUntilCount(1);
             // we will block search
             lsmBtree.clearSearchCallbacks();
@@ -704,7 +739,13 @@ public class ComponentRollbackTest {
                     ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
                     try {
                         dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
-                        ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+                        ILSMComponentId next =
+                                dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+                        long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+                        Map<String, Object> flushMap = new HashMap<>();
+                        flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+                        flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+                        lsmAccessor.getOpContext().setParameters(flushMap);
                         lsmAccessor.deleteComponents(predicate);
                     } catch (HyracksDataException e) {
                         failure = e;
@@ -733,9 +774,8 @@ public class ComponentRollbackTest {
         @Override
         public boolean test(ILSMComponent c) {
             try {
-                return c instanceof ILSMMemoryComponent
-                        || (c instanceof ILSMDiskComponent && AbstractLSMIOOperationCallback
-                                .getTreeIndexLSN(((ILSMDiskComponent) c).getMetadata()) >= lsn);
+                return c instanceof ILSMMemoryComponent || (c instanceof ILSMDiskComponent
+                        && LSMIOOperationCallback.getTreeIndexLSN(((ILSMDiskComponent) c).getMetadata()) >= lsn);
             } catch (HyracksDataException e) {
                 e.printStackTrace();
                 return false;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
index 8bafd32..eb16cf4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
@@ -91,7 +91,7 @@ public class IoCallbackFailureTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(ILSMMemoryComponent c) throws HyracksDataException {
                 // No Op
             }
         };

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index dabb32c..e4623fd 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.test.dataflow;
 
 import java.io.File;
+import java.lang.reflect.Field;
 import java.rmi.RemoteException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -60,8 +61,14 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
 import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -69,6 +76,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class LSMFlushRecoveryTest {
+    public static final Logger LOGGER = LogManager.getLogger();
     private static TestNodeController nc;
     private static Dataset dataset;
     private static PrimaryIndexInfo[] primaryIndexInfos;
@@ -153,6 +161,22 @@ public class LSMFlushRecoveryTest {
     private void initializeNc(boolean cleanUpOnStart) throws Exception {
         nc.init(cleanUpOnStart);
         ncAppCtx = nc.getAppRuntimeContext();
+        // Override the LSMIOScheduler to avoid halting on failure and enable
+        // testing failure scenario in a unit test setting
+        Field ioScheduler = ncAppCtx.getClass().getDeclaredField("lsmIOScheduler");
+        ioScheduler.setAccessible(true);
+        ioScheduler.set(ncAppCtx, new AsynchronousScheduler(ncAppCtx.getServiceContext().getThreadFactory(),
+                new IIoOperationFailedCallback() {
+                    @Override
+                    public void schedulerFailed(ILSMIOOperationScheduler scheduler, Throwable failure) {
+                        LOGGER.error("Scheduler Failed", failure);
+                    }
+
+                    @Override
+                    public void operationFailed(ILSMIOOperation operation, Throwable t) {
+                        LOGGER.warn("IO Operation failed", t);
+                    }
+                }));
         dsLifecycleMgr = ncAppCtx.getDatasetLifecycleManager();
     }
 
@@ -241,21 +265,21 @@ public class LSMFlushRecoveryTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
 
             }
         });
 
         Semaphore primaryFlushSemaphore = new Semaphore(0);
         primaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
-        primaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+        primaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
             @Override
             public void before(Void t) throws HyracksDataException {
 
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Void t) throws HyracksDataException {
                 primaryFlushSemaphore.release();
             }
         });
@@ -283,21 +307,21 @@ public class LSMFlushRecoveryTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
 
             }
         });
 
         Semaphore secondaryFlushSemaphore = new Semaphore(0);
         secondaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
-        secondaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+        secondaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
             @Override
             public void before(Void t) throws HyracksDataException {
 
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Void t) throws HyracksDataException {
                 secondaryFlushSemaphore.release();
             }
         });
@@ -335,21 +359,21 @@ public class LSMFlushRecoveryTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
 
             }
         });
 
         Semaphore flushSemaphore = new Semaphore(0);
         secondaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
-        secondaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+        secondaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
             @Override
             public void before(Void t) throws HyracksDataException {
 
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Void t) throws HyracksDataException {
                 flushSemaphore.release();
             }
         });
@@ -387,7 +411,7 @@ public class LSMFlushRecoveryTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
 
             }
         });
@@ -400,7 +424,7 @@ public class LSMFlushRecoveryTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
 
             }
         });

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
index 62705cc..1795c93 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
@@ -302,7 +302,7 @@ public class MultiPartitionLSMIndexTest {
                 }
 
                 @Override
-                public void after() {
+                public void after(Void t) {
                     synchronized (allocated) {
                         allocated.setValue(true);
                         allocated.notifyAll();
@@ -339,7 +339,7 @@ public class MultiPartitionLSMIndexTest {
                 }
 
                 @Override
-                public void after() {
+                public void after(Semaphore t) {
                 }
             });
             synchronized (proceedToScheduleFlush) {
@@ -421,7 +421,7 @@ public class MultiPartitionLSMIndexTest {
                 }
 
                 @Override
-                public void after() {
+                public void after(Void t) {
                     synchronized (finishedSchduleFlush) {
                         finishedSchduleFlush.set(true);
                         finishedSchduleFlush.notifyAll();
@@ -478,7 +478,7 @@ public class MultiPartitionLSMIndexTest {
                 }
 
                 @Override
-                public void after() {
+                public void after(ILSMMemoryComponent t) {
                     synchronized (recycledPrimary) {
                         recycledPrimary.setValue(true);
                         recycledPrimary.notifyAll();
@@ -519,7 +519,7 @@ public class MultiPartitionLSMIndexTest {
                 }
 
                 @Override
-                public void after() {
+                public void after(ILSMMemoryComponent t) {
                 }
             };
             secondaryLsmBtrees[0].addIoRecycleCallback(secondaryRecycleCallback);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
index e4373f6..72026a2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
@@ -60,8 +60,9 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
 import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationStatus;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -235,10 +236,11 @@ public class SearchCursorComponentSwitchTest {
             // merge all components
             ILSMIndexAccessor mergeAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
             List<ILSMDiskComponent> mergedComponents = new ArrayList<>(lsmBtree.getDiskComponents());
-            BlockingIOOperationCallbackWrapper ioCallback =
-                    new BlockingIOOperationCallbackWrapper(lsmBtree.getIOOperationCallback());
-            mergeAccessor.scheduleMerge(ioCallback, mergedComponents);
-            ioCallback.waitForIO();
+            ILSMIOOperation merge = mergeAccessor.scheduleMerge(mergedComponents);
+            merge.sync();
+            if (merge.getStatus() == LSMIOOperationStatus.FAILURE) {
+                throw HyracksDataException.create(merge.getFailure());
+            }
             // unblock the search
             unblockSearch(lsmBtree);
             // ensure the search got the correct number

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
index 390286a..d08fc72 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
@@ -253,7 +253,7 @@ public class StorageTestUtils {
                 }
 
                 @Override
-                public void after() {
+                public void after(Semaphore t) {
                 }
             });
             Callable<Boolean> callable = new Callable<Boolean>() {
@@ -292,7 +292,7 @@ public class StorageTestUtils {
                 }
 
                 @Override
-                public void after() {
+                public void after(Semaphore t) {
                 }
             });
         }
@@ -319,7 +319,7 @@ public class StorageTestUtils {
                 }
 
                 @Override
-                public void after() {
+                public void after(Semaphore t) {
                 }
             });
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
index 0c4983a..20875a3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
@@ -62,6 +62,6 @@ public class TestDataset extends Dataset {
 
     @Override
     public ILSMIOOperationCallbackFactory getIoOperationCallbackFactory(Index index) throws AlgebricksException {
-        return new TestLsmBtreeIoOpCallbackFactory(getComponentIdGeneratorFactory());
+        return new TestLsmIoOpCallbackFactory(getComponentIdGeneratorFactory(), getDatasetInfoProvider());
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
deleted file mode 100644
index 5852ad9..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.dataflow;
-
-import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
-import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
-import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGeneratorFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
-import org.apache.hyracks.storage.am.lsm.common.impls.EmptyComponent;
-
-public class TestLsmBtreeIoOpCallbackFactory extends LSMBTreeIOOperationCallbackFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private static volatile int completedFlushes = 0;
-    private static volatile int completedMerges = 0;
-    private static volatile int rollbackFlushes = 0;
-    private static volatile int rollbackMerges = 0;
-    private static volatile int failedFlushes = 0;
-    private static volatile int failedMerges = 0;
-
-    public TestLsmBtreeIoOpCallbackFactory(ILSMComponentIdGeneratorFactory idGeneratorFactory) {
-        super(idGeneratorFactory);
-    }
-
-    @Override
-    public synchronized ILSMIOOperationCallback createIoOpCallback(ILSMIndex index) throws HyracksDataException {
-        completedFlushes = 0;
-        completedMerges = 0;
-        rollbackFlushes = 0;
-        rollbackMerges = 0;
-        // Whenever this is called, it resets the counter
-        // However, the counters for the failed operations are never reset since we expect them
-        // To be always 0
-        return new TestLsmBtreeIoOpCallback(index, getComponentIdGenerator(), getIndexCheckpointManagerProvider());
-    }
-
-    public int getTotalFlushes() {
-        return completedFlushes + rollbackFlushes;
-    }
-
-    public int getTotalMerges() {
-        return completedMerges + rollbackMerges;
-    }
-
-    public int getTotalIoOps() {
-        return getTotalFlushes() + getTotalMerges();
-    }
-
-    public int getRollbackFlushes() {
-        return rollbackFlushes;
-    }
-
-    public int getRollbackMerges() {
-        return rollbackMerges;
-    }
-
-    public int getCompletedFlushes() {
-        return completedFlushes;
-    }
-
-    public int getCompletedMerges() {
-        return completedMerges;
-    }
-
-    public static int getFailedFlushes() {
-        return failedFlushes;
-    }
-
-    public static int getFailedMerges() {
-        return failedMerges;
-    }
-
-    public class TestLsmBtreeIoOpCallback extends LSMBTreeIOOperationCallback {
-        private final TestLsmBtree lsmBtree;
-
-        public TestLsmBtreeIoOpCallback(ILSMIndex index, ILSMComponentIdGenerator idGenerator,
-                IIndexCheckpointManagerProvider checkpointManagerProvider) {
-            super(index, idGenerator, checkpointManagerProvider);
-            lsmBtree = (TestLsmBtree) index;
-        }
-
-        @Override
-        public void beforeOperation(ILSMIndexOperationContext opCtx) throws HyracksDataException {
-            lsmBtree.beforeIoOperationCalled();
-            super.beforeOperation(opCtx);
-            lsmBtree.beforeIoOperationReturned();
-        }
-
-        @Override
-        public void afterOperation(ILSMIndexOperationContext opCtx) throws HyracksDataException {
-            lsmBtree.afterIoOperationCalled();
-            super.afterOperation(opCtx);
-            lsmBtree.afterIoOperationReturned();
-        }
-
-        @Override
-        public void afterFinalize(ILSMIndexOperationContext opCtx) throws HyracksDataException {
-            lsmBtree.afterIoFinalizeCalled();
-            super.afterFinalize(opCtx);
-            synchronized (TestLsmBtreeIoOpCallbackFactory.this) {
-                if (opCtx.getNewComponent() != null) {
-                    if (opCtx.getNewComponent() == EmptyComponent.INSTANCE) {
-                        if (opCtx.getIoOperationType() == LSMIOOperationType.FLUSH) {
-                            rollbackFlushes++;
-                        } else {
-                            rollbackMerges++;
-                        }
-                    } else {
-                        if (opCtx.getIoOperationType() == LSMIOOperationType.FLUSH) {
-                            completedFlushes++;
-                        } else {
-                            completedMerges++;
-                        }
-                    }
-                } else {
-                    recordFailure(opCtx.getIoOperationType());
-                }
-                TestLsmBtreeIoOpCallbackFactory.this.notifyAll();
-            }
-            lsmBtree.afterIoFinalizeReturned();
-        }
-
-        @Override
-        public void recycled(ILSMMemoryComponent component, boolean advance) throws HyracksDataException {
-            lsmBtree.recycledCalled(component);
-            super.recycled(component, advance);
-            lsmBtree.recycledReturned(component);
-        }
-
-        @Override
-        public void allocated(ILSMMemoryComponent component) throws HyracksDataException {
-            lsmBtree.allocatedCalled(component);
-            super.allocated(component);
-            lsmBtree.allocatedReturned(component);
-        }
-
-        private void recordFailure(LSMIOOperationType opType) {
-            if (opType == LSMIOOperationType.FLUSH) {
-                failedFlushes++;
-            } else {
-                failedMerges++;
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
new file mode 100644
index 0000000..c762c8c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import org.apache.asterix.common.api.IDatasetInfoProvider;
+import org.apache.asterix.common.api.ILSMComponentIdGeneratorFactory;
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIndexIOOperationCallbackFactory;
+import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.EmptyComponent;
+
+public class TestLsmIoOpCallbackFactory extends LSMIndexIOOperationCallbackFactory {
+
+    private static final long serialVersionUID = 1L;
+
+    private static volatile int completedFlushes = 0;
+    private static volatile int completedMerges = 0;
+    private static volatile int rollbackFlushes = 0;
+    private static volatile int rollbackMerges = 0;
+    private static volatile int failedFlushes = 0;
+    private static volatile int failedMerges = 0;
+
+    public TestLsmIoOpCallbackFactory(ILSMComponentIdGeneratorFactory idGeneratorFactory,
+            IDatasetInfoProvider datasetInfoProvider) {
+        super(idGeneratorFactory, datasetInfoProvider);
+    }
+
+    @Override
+    public synchronized ILSMIOOperationCallback createIoOpCallback(ILSMIndex index) throws HyracksDataException {
+        completedFlushes = 0;
+        completedMerges = 0;
+        rollbackFlushes = 0;
+        rollbackMerges = 0;
+        // Whenever this is called, it resets the counter
+        // However, the counters for the failed operations are never reset since we expect them
+        // To be always 0
+        return new TestLsmIoOpCallback(datasetInfoProvider.getDatasetInfo(ncCtx), index,
+                getComponentIdGenerator().getId(), getIndexCheckpointManagerProvider());
+    }
+
+    public int getTotalFlushes() {
+        return completedFlushes + rollbackFlushes;
+    }
+
+    public int getTotalMerges() {
+        return completedMerges + rollbackMerges;
+    }
+
+    public int getTotalIoOps() {
+        return getTotalFlushes() + getTotalMerges();
+    }
+
+    public int getRollbackFlushes() {
+        return rollbackFlushes;
+    }
+
+    public int getRollbackMerges() {
+        return rollbackMerges;
+    }
+
+    public int getCompletedFlushes() {
+        return completedFlushes;
+    }
+
+    public int getCompletedMerges() {
+        return completedMerges;
+    }
+
+    public static int getFailedFlushes() {
+        return failedFlushes;
+    }
+
+    public static int getFailedMerges() {
+        return failedMerges;
+    }
+
+    public class TestLsmIoOpCallback extends LSMIOOperationCallback {
+        private final TestLsmBtree lsmBtree;
+
+        public TestLsmIoOpCallback(DatasetInfo dsInfo, ILSMIndex index, ILSMComponentId id,
+                IIndexCheckpointManagerProvider checkpointManagerProvider) {
+            super(dsInfo, index, id, checkpointManagerProvider);
+            lsmBtree = (TestLsmBtree) index;
+        }
+
+        @Override
+        public void scheduled(ILSMIOOperation op) throws HyracksDataException {
+            lsmBtree.ioScheduledCalled();
+            super.scheduled(op);
+            lsmBtree.ioScheduledReturned();
+        }
+
+        @Override
+        public void beforeOperation(ILSMIOOperation op) throws HyracksDataException {
+            lsmBtree.beforeIoOperationCalled();
+            super.beforeOperation(op);
+            lsmBtree.beforeIoOperationReturned();
+        }
+
+        @Override
+        public void afterOperation(ILSMIOOperation op) throws HyracksDataException {
+            lsmBtree.afterIoOperationCalled();
+            super.afterOperation(op);
+            lsmBtree.afterIoOperationReturned();
+        }
+
+        @Override
+        public void afterFinalize(ILSMIOOperation op) throws HyracksDataException {
+            lsmBtree.afterIoFinalizeCalled();
+            super.afterFinalize(op);
+            synchronized (TestLsmIoOpCallbackFactory.this) {
+                if (op.getNewComponent() != null) {
+                    if (op.getNewComponent() == EmptyComponent.INSTANCE) {
+                        if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+                            rollbackFlushes++;
+                        } else {
+                            rollbackMerges++;
+                        }
+                    } else {
+                        if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+                            completedFlushes++;
+                        } else {
+                            completedMerges++;
+                        }
+                    }
+                } else {
+                    recordFailure(op.getIOOpertionType());
+                }
+                TestLsmIoOpCallbackFactory.this.notifyAll();
+            }
+            lsmBtree.afterIoFinalizeReturned();
+        }
+
+        @Override
+        public void completed(ILSMIOOperation operation) {
+            try {
+                lsmBtree.ioCompletedCalled();
+                super.completed(operation);
+                lsmBtree.ioCompletedReturned();
+            } catch (Exception e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
+        @Override
+        public void recycled(ILSMMemoryComponent component) throws HyracksDataException {
+            lsmBtree.recycledCalled(component);
+            super.recycled(component);
+            lsmBtree.recycledReturned(component);
+        }
+
+        @Override
+        public void allocated(ILSMMemoryComponent component) throws HyracksDataException {
+            lsmBtree.allocatedCalled(component);
+            super.allocated(component);
+            lsmBtree.allocatedReturned(component);
+        }
+
+        private void recordFailure(LSMIOOperationType opType) {
+            if (opType == LSMIOOperationType.FLUSH) {
+                failedFlushes++;
+            } else {
+                failedMerges++;
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
index 9a528d3..250c25f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
@@ -60,7 +60,7 @@ public class TestPrimaryIndexOperationTracker extends PrimaryIndexOperationTrack
         super.triggerScheduleFlush(logRecord);
         synchronized (callbacks) {
             for (ITestOpCallback<Void> callback : callbacks) {
-                callback.after();
+                callback.after(null);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
index 06b213d..7a3e475 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
@@ -199,7 +199,7 @@ public class TransactionAbortTest {
             }
 
             @Override
-            public void after() throws HyracksDataException {
+            public void after(Semaphore t) throws HyracksDataException {
                 // manually set the current memory component as modified
                 index.getCurrentMemoryComponent().setModified();
                 throw new HyracksDataException("Fail the job");

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
index 0f6adf6..d5985b5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
@@ -51,7 +51,6 @@ import org.apache.hyracks.api.util.InvokeUtil;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
-import org.apache.hyracks.test.support.TestUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -261,8 +260,9 @@ public class MetadataTxnTest {
         final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction();
         metadataProvider.setMetadataTxnContext(mdTxn);
         final String nodeGroupName = "ng";
+        final String committedNodeGroup = "committed_ng";
+        final List<String> ngNodes = Collections.singletonList("asterix_nc1");
         try {
-            final List<String> ngNodes = Collections.singletonList("asterix_nc1");
             MetadataManager.INSTANCE.addNodegroup(mdTxn, new NodeGroup(nodeGroupName, ngNodes));
             MetadataManager.INSTANCE.commitTransaction(mdTxn);
         } finally {
@@ -281,6 +281,9 @@ public class MetadataTxnTest {
         int diskComponentsBeforeFlush = index.getDiskComponents().size();
         // lock opTracker to prevent log flusher from triggering flush
         synchronized (opTracker) {
+            final MetadataTransactionContext committedMdTxn = MetadataManager.INSTANCE.beginTransaction();
+            MetadataManager.INSTANCE.addNodegroup(committedMdTxn, new NodeGroup(committedNodeGroup, ngNodes));
+            MetadataManager.INSTANCE.commitTransaction(committedMdTxn);
             opTracker.setFlushOnExit(true);
             opTracker.flushIfNeeded();
             Assert.assertTrue(opTracker.isFlushLogCreated());
@@ -288,7 +291,6 @@ public class MetadataTxnTest {
             // make sure force operation will processed
             MetadataManager.INSTANCE.dropNodegroup(mdTxn2, nodeGroupName, false);
             Assert.assertEquals(1, opTracker.getNumActiveOperations());
-            Assert.assertFalse(index.hasFlushRequestForCurrentMutableComponent());
             // release opTracker lock now to allow log flusher to schedule the flush
             InvokeUtil.runWithTimeout(() -> {
                 synchronized (opTracker) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
deleted file mode 100644
index 17509a4..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.storage;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
-import org.apache.asterix.app.bootstrap.TestNodeController;
-import org.apache.asterix.app.data.gen.TupleGenerator;
-import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction;
-import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
-import org.apache.asterix.common.exceptions.ExceptionUtils;
-import org.apache.asterix.common.transactions.ITransactionContext;
-import org.apache.asterix.common.transactions.ITransactionManager;
-import org.apache.asterix.common.transactions.TransactionOptions;
-import org.apache.asterix.external.util.DataflowUtils;
-import org.apache.asterix.file.StorageComponentProvider;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.test.common.TestHelper;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
-import org.apache.hyracks.util.DiskUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class DiskIsFullTest {
-
-    private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
-    private static final ARecordType RECORD_TYPE = new ARecordType("TestRecordType", new String[] { "key", "value" },
-            new IAType[] { BuiltinType.AINT32, BuiltinType.AINT64 }, false);
-    private static final GenerationFunction[] RECORD_GEN_FUNCTION =
-            { GenerationFunction.DETERMINISTIC, GenerationFunction.DETERMINISTIC };
-    private static final boolean[] UNIQUE_RECORD_FIELDS = { true, false };
-    private static final ARecordType META_TYPE = null;
-    private static final GenerationFunction[] META_GEN_FUNCTION = null;
-    private static final boolean[] UNIQUE_META_FIELDS = null;
-    private static final int[] KEY_INDEXES = { 0 };
-    private static final int[] KEY_INDICATOR = { Index.RECORD_INDICATOR };
-    private static final List<Integer> KEY_INDICATOR_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR });
-    private static final int DATASET_ID = 101;
-    private static final String DATAVERSE_NAME = "TestDV";
-    private static final String DATASET_NAME = "TestDS";
-    private static final String DATA_TYPE_NAME = "DUMMY";
-    private static final String NODE_GROUP_NAME = "DEFAULT";
-    private static final String TEST_DISK_NAME = "asterixdb_ram_disk";
-    private boolean shouldRun = true;
-
-    @Before
-    public void setUp() throws Exception {
-        if (!SystemUtils.IS_OS_MAC) {
-            System.out.println("Skipping test " + DiskIsFullTest.class.getName() + " due to unsupported OS");
-            shouldRun = false;
-            return;
-        }
-        System.out.println("SetUp: ");
-        TestHelper.deleteExistingInstanceFiles();
-        // create RAM disk
-        final Path ramDiskRoot = DiskUtil.mountRamDisk(TEST_DISK_NAME, 4, MEGABYTE);
-        // Use RAM disk for storage
-        AsterixHyracksIntegrationUtil.setStoragePath(ramDiskRoot.toAbsolutePath().toString());
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        if (!shouldRun) {
-            return;
-        }
-        System.out.println("TearDown");
-        TestHelper.deleteExistingInstanceFiles();
-        DiskUtil.unmountRamDisk(TEST_DISK_NAME);
-        AsterixHyracksIntegrationUtil.restoreDefaultStoragePath();
-    }
-
-    @Test
-    public void testDiskIsFull() {
-        if (!shouldRun) {
-            return;
-        }
-        HyracksDataException expectedException =
-                HyracksDataException.create(ErrorCode.CANNOT_MODIFY_INDEX_DISK_IS_FULL);
-        try {
-            TestNodeController nc = new TestNodeController(null, false);
-            nc.init();
-            StorageComponentProvider storageManager = new StorageComponentProvider();
-            List<List<String>> partitioningKeys = new ArrayList<>();
-            partitioningKeys.add(Collections.singletonList("key"));
-            Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME,
-                    NoMergePolicyFactory.NAME, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH,
-                            partitioningKeys, null, null, null, false, null),
-                    null, DatasetType.INTERNAL, DATASET_ID, 0);
-            try {
-                nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null, storageManager, KEY_INDEXES,
-                        KEY_INDICATOR_LIST, 0);
-                JobId jobId = nc.newJobId();
-                IHyracksTaskContext ctx = nc.createTestContext(jobId, 0, false);
-                ITransactionContext txnCtx = nc.getTransactionManager().beginTransaction(nc.getTxnJobId(ctx),
-                        new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL));
-                // Prepare insert operation
-                LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES,
-                        RECORD_TYPE, META_TYPE, null, KEY_INDEXES, KEY_INDICATOR_LIST, storageManager, null).getLeft();
-                try {
-                    insertOp.open();
-                    TupleGenerator tupleGenerator =
-                            new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATOR, RECORD_GEN_FUNCTION,
-                                    UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
-                    VSizeFrame frame = new VSizeFrame(ctx);
-                    FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
-                    // Insert records until disk becomes full
-                    int tupleCount = 100000;
-                    while (tupleCount > 0) {
-                        ITupleReference tuple = tupleGenerator.next();
-                        try {
-                            DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
-                        } catch (Throwable t) {
-                            final Throwable rootCause = ExceptionUtils.getRootCause(t);
-                            rootCause.printStackTrace();
-                            if (rootCause instanceof HyracksDataException) {
-                                HyracksDataException cause = (HyracksDataException) rootCause;
-                                Assert.assertEquals(cause.getErrorCode(), expectedException.getErrorCode());
-                                Assert.assertEquals(cause.getMessage(), expectedException.getMessage());
-                                return;
-                            } else {
-                                break;
-                            }
-                        }
-                        tupleCount--;
-                    }
-                    Assert.fail("Expected exception (" + expectedException + ") was not thrown");
-                } finally {
-                    try {
-                        insertOp.close();
-                    } finally {
-                        nc.getTransactionManager().abortTransaction(txnCtx.getTxnId());
-                    }
-                }
-            } finally {
-                nc.deInit();
-            }
-        } catch (Throwable e) {
-            e.printStackTrace();
-            Assert.fail("Expected exception (" + expectedException + ") was not thrown");
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
new file mode 100644
index 0000000..7075739
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.hyracks.api.application.INCServiceContext;
+
+@FunctionalInterface
+public interface IDatasetInfoProvider extends Serializable {
+    /**
+     * Get the dataset info
+     *
+     * @param serviceCtx
+     *            the nc service context
+     * @return the dataset info object
+     */
+    DatasetInfo getDatasetInfo(INCServiceContext serviceCtx);
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
new file mode 100644
index 0000000..5fd37e9
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
+import org.apache.hyracks.storage.common.IResource;
+
+@FunctionalInterface
+public interface ILSMComponentIdGeneratorFactory extends Serializable {
+    /**
+     * Get the Id generator for the resource
+     *
+     * @param serviceCtx
+     * @param resource
+     * @return
+     * @throws HyracksDataException
+     */
+    ILSMComponentIdGenerator getComponentIdGenerator(INCServiceContext serviceCtx, IResource resource)
+            throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index b7af0b6..3ff13cb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -37,8 +37,7 @@ public class BaseOperationTracker implements ITransactionOperationTracker {
     @Override
     public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
             IModificationOperationCallback modificationCallback) throws HyracksDataException {
-        if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE
-                || opType == LSMOperationType.REPLICATE) {
+        if (opType == LSMOperationType.REPLICATE) {
             dsInfo.declareActiveIOOperation();
         }
     }
@@ -47,14 +46,14 @@ public class BaseOperationTracker implements ITransactionOperationTracker {
     public void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
             IModificationOperationCallback modificationCallback) throws HyracksDataException {
         if (opType == LSMOperationType.REPLICATE) {
-            dsInfo.undeclareActiveIOOperation();
+            completeOperation(index, opType, searchCallback, modificationCallback);
         }
     }
 
     @Override
     public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
             IModificationOperationCallback modificationCallback) throws HyracksDataException {
-        if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE) {
+        if (opType == LSMOperationType.REPLICATE) {
             dsInfo.undeclareActiveIOOperation();
         }
     }