You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/04/30 20:50:43 UTC
[8/8] asterixdb git commit: [NO ISSUE][STO] Add consistency to flush
lifecycle
[NO ISSUE][STO] Add consistency to flush lifecycle
- user model changes: no
- storage format changes: yes
- renamed AbstractLSMIOOperationCallbackFactory
to LSMIOOperationCallbackFactory
- useless classes have been removed.
- LSMBTreeIOOperationCallbackFactory
- LSMBTreeWithBuddyIOOperationCallbackFactory
- LSMInvertedIndexIOOperationCallbackFactory
- LSMRTreeIOOperationCallbackFactory
- interface changes: yes
Details:
- Previously, flushes have different lifecycle depending
on the memory component state
- not allocated
- allocated
- modified
- In certain cases, flush operations are skipped alltogether
- IO Operation callbacks became complicated and difficult
to maintain since calls are done differently in different
cases.
- In certain cases, afterFinalize is called on the IO
Operation callbacks even if beforeOperation was never
called.
- In this change, flushes go through the same lifecycle
events regardless of the state of the memory component.
- In addition, primary and secondary memory components
would reside in different virtual buffer caches due
to skipped flushes, or due to having the secondary
index created when the primary index's memory component
is residing on the virtual buffer cache with index !=0.
- Moreover, when flushes are lagging and all memory
components are being flushed, search operations assumes
the oldest of the memory component is the newest and
produces incorrect results.
- In addition, in case of a failed flush of a component,
the IO scheduler would skip it and flush the next
component. This would produce a bad state on disk.
- In this change, a failed flush can be retried. otherwise,
all future flushes of the component fail due to the failure
of the previously failed flush.
- Previously, when a component fails to modify an index due
to flush failures, it assumes disk is full.
- With this change, the modification failure reports the
original cause of the failed flush.
Change-Id: I29f7992ec6c0f71c5b63d45800b2fb590d651e4b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2584
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/4ed0eac5
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/4ed0eac5
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/4ed0eac5
Branch: refs/heads/master
Commit: 4ed0eac5cb4cf5b849bc65847bd4e979d01a210f
Parents: 9410ddd
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Sat Apr 28 18:26:08 2018 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Sun Apr 29 09:48:55 2018 -0700
----------------------------------------------------------------------
.../org/apache/asterix/app/nc/HaltCallback.java | 49 ++
.../asterix/app/nc/NCAppRuntimeContext.java | 11 +-
.../apache/asterix/app/nc/RecoveryManager.java | 27 +-
.../test/dataflow/ComponentRollbackTest.java | 82 +++-
.../test/dataflow/IoCallbackFailureTest.java | 2 +-
.../test/dataflow/LSMFlushRecoveryTest.java | 46 +-
.../dataflow/MultiPartitionLSMIndexTest.java | 10 +-
.../SearchCursorComponentSwitchTest.java | 12 +-
.../asterix/test/dataflow/StorageTestUtils.java | 6 +-
.../asterix/test/dataflow/TestDataset.java | 2 +-
.../TestLsmBtreeIoOpCallbackFactory.java | 170 -------
.../dataflow/TestLsmIoOpCallbackFactory.java | 192 ++++++++
.../TestPrimaryIndexOperationTracker.java | 2 +-
.../test/dataflow/TransactionAbortTest.java | 2 +-
.../asterix/test/metadata/MetadataTxnTest.java | 8 +-
.../asterix/test/storage/DiskIsFullTest.java | 182 -------
.../common/api/IDatasetInfoProvider.java | 36 ++
.../api/ILSMComponentIdGeneratorFactory.java | 40 ++
.../common/context/BaseOperationTracker.java | 7 +-
.../context/CorrelatedPrefixMergePolicy.java | 2 +-
.../common/context/DatasetInfoProvider.java | 42 ++
.../DatasetLSMComponentIdGeneratorFactory.java | 3 +-
.../common/context/DatasetLifecycleManager.java | 46 +-
.../context/PrimaryIndexOperationTracker.java | 56 ++-
.../asterix/common/dataflow/LSMIndexUtil.java | 12 +-
.../common/exceptions/ExceptionUtils.java | 21 +-
.../AbstractLSMIOOperationCallback.java | 287 -----------
...tractLSMIndexIOOperationCallbackFactory.java | 86 ----
.../LSMBTreeIOOperationCallback.java | 32 --
.../LSMBTreeIOOperationCallbackFactory.java | 39 --
.../LSMBTreeWithBuddyIOOperationCallback.java | 31 --
...TreeWithBuddyIOOperationCallbackFactory.java | 39 --
.../ioopcallbacks/LSMIOOperationCallback.java | 227 +++++++++
.../LSMIndexIOOperationCallbackFactory.java | 76 +++
.../LSMInvertedIndexIOOperationCallback.java | 32 --
...InvertedIndexIOOperationCallbackFactory.java | 40 --
.../LSMRTreeIOOperationCallback.java | 32 --
.../LSMRTreeIOOperationCallbackFactory.java | 39 --
.../asterix/common/utils/StorageConstants.java | 2 +-
.../CorrelatedPrefixMergePolicyTest.java | 6 +-
.../AbstractLSMIOOperationCallbackTest.java | 315 -------------
.../LSMBTreeIOOperationCallbackTest.java | 39 --
...SMBTreeWithBuddyIOOperationCallbackTest.java | 39 --
.../LSMIOOperationCallbackTest.java | 217 +++++++++
...LSMInvertedIndexIOOperationCallbackTest.java | 39 --
.../LSMRTreeIOOperationCallbackTest.java | 39 --
.../test/ioopcallbacks/TestFlushOperation.java | 56 +++
.../ioopcallbacks/TestLSMIndexAccessor.java | 183 ++++++++
.../TestLSMIndexOperationContext.java | 25 +-
.../metadata/api/IActiveEntityController.java | 2 +-
.../metadata/bootstrap/MetadataBootstrap.java | 11 +-
.../asterix/metadata/entities/Dataset.java | 33 +-
.../replication/sync/IndexSynchronizer.java | 6 +-
...irstRuleCheckFixpointRuleControllerTest.java | 2 +
.../hyracks/api/exceptions/ErrorCode.java | 2 +-
.../apache/hyracks/api/util/ExceptionUtils.java | 11 +
.../api/util/SingleThreadEventProcessor.java | 10 +-
.../src/main/resources/errormsg/en.properties | 2 +-
.../am/common/ophelpers/IndexOperation.java | 3 +-
.../am/lsm/btree/impls/ExternalBTree.java | 26 +-
.../lsm/btree/impls/ExternalBTreeWithBuddy.java | 32 +-
.../storage/am/lsm/btree/impls/LSMBTree.java | 17 +-
.../btree/impls/LSMBTreeMemoryComponent.java | 4 +-
.../am/lsm/btree/impls/LSMBTreeOpContext.java | 2 -
.../btree/impls/LSMBTreeRangeSearchCursor.java | 5 +-
.../impls/LSMBTreeWithBuddyMemoryComponent.java | 6 +-
.../am/lsm/btree/utils/LSMBTreeUtil.java | 5 -
...AbstractLSMWithBloomFilterDiskComponent.java | 5 +-
.../AbstractLSMWithBuddyMemoryComponent.java | 63 ---
.../common/api/IIoOperationFailedCallback.java | 44 ++
.../am/lsm/common/api/ILSMComponent.java | 9 +
.../common/api/ILSMComponentIdGenerator.java | 9 +-
.../api/ILSMComponentIdGeneratorFactory.java | 31 --
.../am/lsm/common/api/ILSMDiskComponent.java | 5 +-
.../common/api/ILSMDiskComponentBulkLoader.java | 5 +
.../storage/am/lsm/common/api/ILSMHarness.java | 12 +-
.../am/lsm/common/api/ILSMIOOperation.java | 65 ++-
.../lsm/common/api/ILSMIOOperationCallback.java | 33 +-
.../api/ILSMIOOperationCallbackFactory.java | 13 +
.../storage/am/lsm/common/api/ILSMIndex.java | 36 +-
.../am/lsm/common/api/ILSMIndexAccessor.java | 24 +-
.../common/api/ILSMIndexOperationContext.java | 25 +-
.../am/lsm/common/api/ILSMMemoryComponent.java | 21 +-
.../LSMIndexCompactOperatorNodePushable.java | 2 +-
.../lsm/common/impls/AbstractIoOperation.java | 53 +++
.../common/impls/AbstractLSMDiskComponent.java | 41 +-
.../am/lsm/common/impls/AbstractLSMIndex.java | 297 ++++++++----
.../impls/AbstractLSMIndexOperationContext.java | 24 +-
.../impls/AbstractLSMMemoryComponent.java | 118 +++--
.../AbstractLSMWithBuddyMemoryComponent.java | 63 +++
.../lsm/common/impls/AsynchronousScheduler.java | 126 +++--
.../BlockingIOOperationCallbackWrapper.java | 69 ---
.../ChainedLSMDiskComponentBulkLoader.java | 15 +-
.../impls/ComponentReplacementContext.java | 12 +-
.../lsm/common/impls/ConstantMergePolicy.java | 6 +-
.../am/lsm/common/impls/EmptyComponent.java | 8 +-
.../lsm/common/impls/ExternalIndexHarness.java | 78 +--
.../am/lsm/common/impls/FlushOperation.java | 4 +-
.../lsm/common/impls/IoOperationExecutor.java | 121 +++++
.../common/impls/LSMComponentIdGenerator.java | 13 +-
.../impls/LSMComponentIdGeneratorFactory.java | 40 --
.../storage/am/lsm/common/impls/LSMHarness.java | 354 ++++++--------
.../impls/LSMIndexDiskComponentBulkLoader.java | 48 +-
.../lsm/common/impls/LSMTreeIndexAccessor.java | 18 +-
.../am/lsm/common/impls/LoadOperation.java | 49 ++
.../am/lsm/common/impls/MergeOperation.java | 4 +-
.../impls/MultitenantVirtualBufferCache.java | 4 +
.../impls/NoOpIOOperationCallbackFactory.java | 36 +-
.../am/lsm/common/impls/NoOpIoOperation.java | 116 +++++
.../impls/NoOpIoOperationFailedCallback.java | 44 ++
.../am/lsm/common/impls/PrefixMergePolicy.java | 7 +-
.../common/impls/StubIOOperationCallback.java | 70 ---
.../lsm/common/impls/SynchronousScheduler.java | 77 ++-
.../impls/SynchronousSchedulerProvider.java | 10 +-
.../lsm/common/impls/ThreadCountingTracker.java | 2 +-
.../am/lsm/common/impls/TracedIOOperation.java | 43 +-
.../am/lsm/common/impls/VirtualBufferCache.java | 6 +
.../am/lsm/common/util/IOOperationUtils.java | 43 --
.../invertedindex/impls/LSMInvertedIndex.java | 20 +-
.../impls/LSMInvertedIndexAccessor.java | 18 +-
.../impls/LSMInvertedIndexMemoryComponent.java | 6 +-
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 5 +-
.../am/lsm/rtree/impls/ExternalRTree.java | 27 +-
.../storage/am/lsm/rtree/impls/LSMRTree.java | 17 +-
.../rtree/impls/LSMRTreeMemoryComponent.java | 6 +-
.../impls/LSMRTreeWithAntiMatterTuples.java | 10 +-
.../storage/common/IIndexBulkLoader.java | 2 +-
.../storage/common/buffercache/BufferCache.java | 19 +
.../common/buffercache/DebugBufferCache.java | 6 +
.../common/buffercache/IBufferCache.java | 9 +
.../am/lsm/btree/EncapsulatingIoCallback.java | 107 +++++
.../btree/EncapsulatingIoCallbackFactory.java | 71 +++
.../btree/LSMBTreeComponentLifecycleTest.java | 470 +++++++++++++++++++
.../am/lsm/btree/LSMBTreeFileManagerTest.java | 5 +-
.../btree/LSMBTreeFilterMergeTestDriver.java | 18 +-
.../am/lsm/btree/LSMBTreeMergeFailTest.java | 39 +-
.../am/lsm/btree/LSMBTreeMergeTestDriver.java | 3 +-
...MBTreeModificationOperationCallbackTest.java | 21 +-
.../btree/LSMBTreeScanDiskComponentsTest.java | 9 +-
...TreeUpdateInPlaceScanDiskComponentsTest.java | 5 +-
.../am/lsm/btree/LSMBTreeUpdateInPlaceTest.java | 16 +-
.../am/lsm/btree/impl/AllowTestOpCallback.java | 2 +-
.../btree/impl/CountingIoOperationCallback.java | 101 ++++
.../CountingIoOperationCallbackFactory.java | 48 ++
.../am/lsm/btree/impl/ITestOpCallback.java | 2 +-
.../am/lsm/btree/impl/NoOpTestCallback.java | 45 ++
.../storage/am/lsm/btree/impl/TestLsmBtree.java | 87 +++-
.../lsm/btree/impl/TestVirtualBufferCache.java | 6 +
.../btree/multithread/LSMBTreeTestWorker.java | 2 +-
.../am/lsm/btree/perf/LSMTreeRunner.java | 19 +-
.../am/lsm/btree/util/LSMBTreeTestHarness.java | 8 +-
.../lsm/common/test/PrefixMergePolicyTest.java | 6 +-
.../LSMInvertedIndexMergeTest.java | 3 +-
.../PartitionedLSMInvertedIndexMergeTest.java | 3 +-
.../common/LSMInvertedIndexTestHarness.java | 4 +-
.../multithread/LSMInvertedIndexTestWorker.java | 2 +-
.../am/lsm/rtree/LSMRTreeMergeTestDriver.java | 3 +-
.../LSMRTreeWithAntiMatterTuplesDeleteTest.java | 4 +-
.../rtree/multithread/LSMRTreeTestWorker.java | 2 +-
.../LSMRTreeWithAntiMatterTuplesTestWorker.java | 3 +-
.../am/lsm/rtree/util/LSMRTreeTestHarness.java | 4 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 5 +-
162 files changed, 3954 insertions(+), 2902 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
new file mode 100644
index 0000000..db34131
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/HaltCallback.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc;
+
+import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.util.ExitUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class HaltCallback implements IIoOperationFailedCallback {
+ private static final Logger LOGGER = LogManager.getLogger();
+ public static final HaltCallback INSTANCE = new HaltCallback();
+
+ private HaltCallback() {
+ }
+
+ @Override
+ public void schedulerFailed(ILSMIOOperationScheduler scheduler, Throwable failure) {
+ LOGGER.error("IO Scheduler has failed", failure);
+ ExitUtil.halt(ExitUtil.EC_IO_SCHEDULER_FAILED);
+ }
+
+ @Override
+ public void operationFailed(ILSMIOOperation operation, Throwable t) {
+ LOGGER.error("Operation {} has failed", t);
+ if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ ExitUtil.halt(ExitUtil.EC_FLUSH_FAILED);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 288e5f2..c3b1bae 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -52,7 +52,6 @@ import org.apache.asterix.common.config.TransactionProperties;
import org.apache.asterix.common.context.DatasetLifecycleManager;
import org.apache.asterix.common.context.DatasetMemoryManager;
import org.apache.asterix.common.context.IStorageComponentProvider;
-import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.replication.IReplicationChannel;
@@ -169,7 +168,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
}
@Override
- public void initialize(boolean initialRun) throws IOException, ACIDException {
+ public void initialize(boolean initialRun) throws IOException {
ioManager = getServiceContext().getIoManager();
threadExecutor =
MaintainedThreadNameExecutorService.newCachedThreadPool(getServiceContext().getThreadFactory());
@@ -177,19 +176,13 @@ public class NCAppRuntimeContext implements INcApplicationContext {
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
-
- AsynchronousScheduler.INSTANCE.init(getServiceContext().getThreadFactory());
- lsmIOScheduler = AsynchronousScheduler.INSTANCE;
-
+ lsmIOScheduler = new AsynchronousScheduler(getServiceContext().getThreadFactory(), HaltCallback.INSTANCE);
metadataMergePolicyFactory = new PrefixMergePolicyFactory();
indexCheckpointManagerProvider = new IndexCheckpointManagerProvider(ioManager);
-
ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
new PersistentLocalResourceRepositoryFactory(ioManager, indexCheckpointManagerProvider);
-
localResourceRepository =
(PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
-
txnSubsystem = new TransactionSubsystem(this);
IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
SystemState systemState = recoveryMgr.getSystemState();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index d4e652d..efd173f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -48,7 +48,7 @@ import org.apache.asterix.common.context.DatasetInfo;
import org.apache.asterix.common.context.IndexInfo;
import org.apache.asterix.common.dataflow.DatasetLocalResource;
import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
import org.apache.asterix.common.storage.DatasetResourceReference;
import org.apache.asterix.common.storage.IIndexCheckpointManager;
import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
@@ -401,13 +401,12 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
if (iInfo.isOpen() && iInfo.getPartition() == partition) {
maxDiskLastLsn = resourceId2MaxLSNMap.get(iInfo.getResourceId());
index = iInfo.getIndex();
- AbstractLSMIOOperationCallback ioCallback =
- (AbstractLSMIOOperationCallback) index.getIOOperationCallback();
+ LSMIOOperationCallback ioCallback =
+ (LSMIOOperationCallback) index.getIOOperationCallback();
if (logRecord.getLSN() > maxDiskLastLsn
&& !index.isCurrentMutableComponentEmpty()) {
// schedule flush
- ioCallback.updateLastLSN(logRecord.getLSN());
- redoFlush(index, logRecord);
+ redoFlush(index, logRecord, idGenerator.getId());
redoCount++;
} else {
if (index.isMemoryComponentsAllocated()) {
@@ -417,7 +416,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
} else {
// otherwise, we refresh the id stored in ioCallback
// to ensure the memory component receives correct Id upon activation
- ioCallback.forceRefreshNextId();
+ ioCallback.forceRefreshNextId(idGenerator.getId());
}
}
}
@@ -473,10 +472,10 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
long minFirstLSN = logMgr.getAppendLSN();
if (!openIndexList.isEmpty()) {
for (IIndex index : openIndexList) {
- AbstractLSMIOOperationCallback ioCallback =
- (AbstractLSMIOOperationCallback) ((ILSMIndex) index).getIOOperationCallback();
+ LSMIOOperationCallback ioCallback =
+ (LSMIOOperationCallback) ((ILSMIndex) index).getIOOperationCallback();
if (!((AbstractLSMIndex) index).isCurrentMutableComponentEmpty() || ioCallback.hasPendingFlush()) {
- firstLSN = ioCallback.getFirstLSN();
+ firstLSN = ioCallback.getPersistenceLsn();
minFirstLSN = Math.min(minFirstLSN, firstLSN);
}
}
@@ -823,11 +822,17 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
}
}
- private static void redoFlush(ILSMIndex index, ILogRecord logRecord) throws HyracksDataException {
+ private static void redoFlush(ILSMIndex index, ILogRecord logRecord, ILSMComponentId nextId)
+ throws HyracksDataException {
+ long flushLsn = logRecord.getLSN();
+ Map<String, Object> flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE);
+ accessor.getOpContext().setParameters(flushMap);
long minId = logRecord.getFlushingComponentMinId();
long maxId = logRecord.getFlushingComponentMaxId();
ILSMComponentId id = new LSMComponentId(minId, maxId);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, nextId);
if (!index.getDiskComponents().isEmpty()) {
ILSMDiskComponent diskComponent = index.getDiskComponents().get(0);
ILSMComponentId maxDiskComponentId = diskComponent.getId();
@@ -837,7 +842,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
}
}
index.getCurrentMemoryComponent().resetId(id, true);
- accessor.scheduleFlush(index.getIOOperationCallback());
+ accessor.scheduleFlush();
}
private class JobEntityCommits {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
index 9ef531e..a33bda1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
@@ -20,7 +20,9 @@ package org.apache.asterix.test.dataflow;
import java.io.File;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.function.Predicate;
import org.apache.asterix.app.bootstrap.TestNodeController;
@@ -29,7 +31,7 @@ import org.apache.asterix.app.data.gen.TupleGenerator;
import org.apache.asterix.app.nc.NCAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
-import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
import org.apache.asterix.common.transactions.ITransactionContext;
import org.apache.asterix.common.transactions.ITransactionManager;
import org.apache.asterix.common.transactions.TransactionOptions;
@@ -50,6 +52,7 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
@@ -146,16 +149,27 @@ public class ComponentRollbackTest {
StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ ILSMComponentId next =
+ dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ Map<String, Object> flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
// rollback a memory component
lsmAccessor.deleteComponents(memoryComponentsPredicate);
StorageTestUtils.searchAndAssertCount(nc, PARTITION,
StorageTestUtils.TOTAL_NUM_OF_RECORDS - StorageTestUtils.RECORDS_PER_COMPONENT);
// rollback the last disk component
lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
lsmAccessor.deleteComponents(pred);
StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -203,7 +217,13 @@ public class ComponentRollbackTest {
StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ ILSMComponentId next =
+ dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ Map<String, Object> flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
// rollback a memory component
lsmAccessor.deleteComponents(memoryComponentsPredicate);
StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -227,9 +247,14 @@ public class ComponentRollbackTest {
StorageTestUtils.searchAndAssertCount(nc, PARTITION, StorageTestUtils.TOTAL_NUM_OF_RECORDS);
// rollback the last disk component
lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
lsmAccessor.deleteComponents(pred);
StorageTestUtils.searchAndAssertCount(nc, PARTITION,
@@ -278,7 +303,13 @@ public class ComponentRollbackTest {
// now that we enetered, we will rollback
ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ ILSMComponentId next =
+ dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ Map<String, Object> flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
// rollback a memory component
lsmAccessor.deleteComponents(
c -> (c instanceof ILSMMemoryComponent && ((ILSMMemoryComponent) c).isModified()));
@@ -297,10 +328,14 @@ public class ComponentRollbackTest {
// wait till firstSearcher enter the components
secondSearcher.waitUntilEntered();
lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
-
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ next = dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
DiskComponentLsnPredicate pred = new DiskComponentLsnPredicate(lsn);
lsmAccessor.deleteComponents(pred);
// now that the rollback has completed, we will unblock the search
@@ -407,11 +442,11 @@ public class ComponentRollbackTest {
// select the components to merge... the last three
int numMergedComponents = 3;
List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
for (int i = 0; i < numMergedComponents; i++) {
mergedComponents.add(diskComponents.get(i));
}
- mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+ mergeAccessor.scheduleMerge(mergedComponents);
merger.waitUntilCount(1);
// now that we enetered, we will rollback
Rollerback rollerback = new Rollerback(lsmBtree, new DiskComponentLsnPredicate(lsn));
@@ -590,11 +625,11 @@ public class ComponentRollbackTest {
// select the components to merge... the last three
int numMergedComponents = 3;
List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
for (int i = 0; i < numMergedComponents; i++) {
mergedComponents.add(diskComponents.get(i));
}
- mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+ mergeAccessor.scheduleMerge(mergedComponents);
merger.waitUntilCount(1);
// we will block search
lsmBtree.clearSearchCallbacks();
@@ -659,12 +694,12 @@ public class ComponentRollbackTest {
ILSMIndexAccessor mergeAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
// select the components to merge... the last three
List<ILSMDiskComponent> mergedComponents = new ArrayList<>();
- long lsn = AbstractLSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
+ long lsn = LSMIOOperationCallback.getTreeIndexLSN(diskComponents.get(0).getMetadata());
int numMergedComponents = 3;
for (int i = 0; i < numMergedComponents; i++) {
mergedComponents.add(diskComponents.get(i));
}
- mergeAccessor.scheduleMerge(lsmBtree.getIOOperationCallback(), mergedComponents);
+ mergeAccessor.scheduleMerge(mergedComponents);
merger.waitUntilCount(1);
// we will block search
lsmBtree.clearSearchCallbacks();
@@ -704,7 +739,13 @@ public class ComponentRollbackTest {
ILSMIndexAccessor lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
try {
dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).refresh();
- ((AbstractLSMIOOperationCallback) lsmBtree.getIOOperationCallback()).updateLastLSN(0);
+ ILSMComponentId next =
+ dsLifecycleMgr.getComponentIdGenerator(StorageTestUtils.DATASET_ID, PARTITION).getId();
+ long flushLsn = nc.getTransactionSubsystem().getLogManager().getAppendLSN();
+ Map<String, Object> flushMap = new HashMap<>();
+ flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn);
+ flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, next);
+ lsmAccessor.getOpContext().setParameters(flushMap);
lsmAccessor.deleteComponents(predicate);
} catch (HyracksDataException e) {
failure = e;
@@ -733,9 +774,8 @@ public class ComponentRollbackTest {
@Override
public boolean test(ILSMComponent c) {
try {
- return c instanceof ILSMMemoryComponent
- || (c instanceof ILSMDiskComponent && AbstractLSMIOOperationCallback
- .getTreeIndexLSN(((ILSMDiskComponent) c).getMetadata()) >= lsn);
+ return c instanceof ILSMMemoryComponent || (c instanceof ILSMDiskComponent
+ && LSMIOOperationCallback.getTreeIndexLSN(((ILSMDiskComponent) c).getMetadata()) >= lsn);
} catch (HyracksDataException e) {
e.printStackTrace();
return false;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
index 8bafd32..eb16cf4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/IoCallbackFailureTest.java
@@ -91,7 +91,7 @@ public class IoCallbackFailureTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(ILSMMemoryComponent c) throws HyracksDataException {
// No Op
}
};
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
index dabb32c..e4623fd 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LSMFlushRecoveryTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.dataflow;
import java.io.File;
+import java.lang.reflect.Field;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -60,8 +61,14 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.ITestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.IIoOperationFailedCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -69,6 +76,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class LSMFlushRecoveryTest {
+ public static final Logger LOGGER = LogManager.getLogger();
private static TestNodeController nc;
private static Dataset dataset;
private static PrimaryIndexInfo[] primaryIndexInfos;
@@ -153,6 +161,22 @@ public class LSMFlushRecoveryTest {
private void initializeNc(boolean cleanUpOnStart) throws Exception {
nc.init(cleanUpOnStart);
ncAppCtx = nc.getAppRuntimeContext();
+ // Override the LSMIOScheduler to avoid halting on failure and enable
+ // testing failure scenario in a unit test setting
+ Field ioScheduler = ncAppCtx.getClass().getDeclaredField("lsmIOScheduler");
+ ioScheduler.setAccessible(true);
+ ioScheduler.set(ncAppCtx, new AsynchronousScheduler(ncAppCtx.getServiceContext().getThreadFactory(),
+ new IIoOperationFailedCallback() {
+ @Override
+ public void schedulerFailed(ILSMIOOperationScheduler scheduler, Throwable failure) {
+ LOGGER.error("Scheduler Failed", failure);
+ }
+
+ @Override
+ public void operationFailed(ILSMIOOperation operation, Throwable t) {
+ LOGGER.warn("IO Operation failed", t);
+ }
+ }));
dsLifecycleMgr = ncAppCtx.getDatasetLifecycleManager();
}
@@ -241,21 +265,21 @@ public class LSMFlushRecoveryTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
}
});
Semaphore primaryFlushSemaphore = new Semaphore(0);
primaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
- primaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+ primaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
@Override
public void before(Void t) throws HyracksDataException {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Void t) throws HyracksDataException {
primaryFlushSemaphore.release();
}
});
@@ -283,21 +307,21 @@ public class LSMFlushRecoveryTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
}
});
Semaphore secondaryFlushSemaphore = new Semaphore(0);
secondaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
- secondaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+ secondaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
@Override
public void before(Void t) throws HyracksDataException {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Void t) throws HyracksDataException {
secondaryFlushSemaphore.release();
}
});
@@ -335,21 +359,21 @@ public class LSMFlushRecoveryTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
}
});
Semaphore flushSemaphore = new Semaphore(0);
secondaryIndexes[PARTITION_0].addFlushCallback(AllowTestOpCallback.INSTANCE);
- secondaryIndexes[PARTITION_0].addIoAfterFinalizeCallback(new ITestOpCallback<Void>() {
+ secondaryIndexes[PARTITION_0].addIoCompletedCallback(new ITestOpCallback<Void>() {
@Override
public void before(Void t) throws HyracksDataException {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Void t) throws HyracksDataException {
flushSemaphore.release();
}
});
@@ -387,7 +411,7 @@ public class LSMFlushRecoveryTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
}
});
@@ -400,7 +424,7 @@ public class LSMFlushRecoveryTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
}
});
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
index 62705cc..1795c93 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/MultiPartitionLSMIndexTest.java
@@ -302,7 +302,7 @@ public class MultiPartitionLSMIndexTest {
}
@Override
- public void after() {
+ public void after(Void t) {
synchronized (allocated) {
allocated.setValue(true);
allocated.notifyAll();
@@ -339,7 +339,7 @@ public class MultiPartitionLSMIndexTest {
}
@Override
- public void after() {
+ public void after(Semaphore t) {
}
});
synchronized (proceedToScheduleFlush) {
@@ -421,7 +421,7 @@ public class MultiPartitionLSMIndexTest {
}
@Override
- public void after() {
+ public void after(Void t) {
synchronized (finishedSchduleFlush) {
finishedSchduleFlush.set(true);
finishedSchduleFlush.notifyAll();
@@ -478,7 +478,7 @@ public class MultiPartitionLSMIndexTest {
}
@Override
- public void after() {
+ public void after(ILSMMemoryComponent t) {
synchronized (recycledPrimary) {
recycledPrimary.setValue(true);
recycledPrimary.notifyAll();
@@ -519,7 +519,7 @@ public class MultiPartitionLSMIndexTest {
}
@Override
- public void after() {
+ public void after(ILSMMemoryComponent t) {
}
};
secondaryLsmBtrees[0].addIoRecycleCallback(secondaryRecycleCallback);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
index e4373f6..72026a2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/SearchCursorComponentSwitchTest.java
@@ -60,8 +60,9 @@ import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
import org.apache.hyracks.storage.am.lsm.btree.impl.AllowTestOpCallback;
import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationStatus;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
import org.junit.After;
import org.junit.AfterClass;
@@ -235,10 +236,11 @@ public class SearchCursorComponentSwitchTest {
// merge all components
ILSMIndexAccessor mergeAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
List<ILSMDiskComponent> mergedComponents = new ArrayList<>(lsmBtree.getDiskComponents());
- BlockingIOOperationCallbackWrapper ioCallback =
- new BlockingIOOperationCallbackWrapper(lsmBtree.getIOOperationCallback());
- mergeAccessor.scheduleMerge(ioCallback, mergedComponents);
- ioCallback.waitForIO();
+ ILSMIOOperation merge = mergeAccessor.scheduleMerge(mergedComponents);
+ merge.sync();
+ if (merge.getStatus() == LSMIOOperationStatus.FAILURE) {
+ throw HyracksDataException.create(merge.getFailure());
+ }
// unblock the search
unblockSearch(lsmBtree);
// ensure the search got the correct number
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
index 390286a..d08fc72 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/StorageTestUtils.java
@@ -253,7 +253,7 @@ public class StorageTestUtils {
}
@Override
- public void after() {
+ public void after(Semaphore t) {
}
});
Callable<Boolean> callable = new Callable<Boolean>() {
@@ -292,7 +292,7 @@ public class StorageTestUtils {
}
@Override
- public void after() {
+ public void after(Semaphore t) {
}
});
}
@@ -319,7 +319,7 @@ public class StorageTestUtils {
}
@Override
- public void after() {
+ public void after(Semaphore t) {
}
});
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
index 0c4983a..20875a3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestDataset.java
@@ -62,6 +62,6 @@ public class TestDataset extends Dataset {
@Override
public ILSMIOOperationCallbackFactory getIoOperationCallbackFactory(Index index) throws AlgebricksException {
- return new TestLsmBtreeIoOpCallbackFactory(getComponentIdGeneratorFactory());
+ return new TestLsmIoOpCallbackFactory(getComponentIdGeneratorFactory(), getDatasetInfoProvider());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
deleted file mode 100644
index 5852ad9..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmBtreeIoOpCallbackFactory.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.dataflow;
-
-import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback;
-import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
-import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGeneratorFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
-import org.apache.hyracks.storage.am.lsm.common.impls.EmptyComponent;
-
-public class TestLsmBtreeIoOpCallbackFactory extends LSMBTreeIOOperationCallbackFactory {
-
- private static final long serialVersionUID = 1L;
-
- private static volatile int completedFlushes = 0;
- private static volatile int completedMerges = 0;
- private static volatile int rollbackFlushes = 0;
- private static volatile int rollbackMerges = 0;
- private static volatile int failedFlushes = 0;
- private static volatile int failedMerges = 0;
-
- public TestLsmBtreeIoOpCallbackFactory(ILSMComponentIdGeneratorFactory idGeneratorFactory) {
- super(idGeneratorFactory);
- }
-
- @Override
- public synchronized ILSMIOOperationCallback createIoOpCallback(ILSMIndex index) throws HyracksDataException {
- completedFlushes = 0;
- completedMerges = 0;
- rollbackFlushes = 0;
- rollbackMerges = 0;
- // Whenever this is called, it resets the counter
- // However, the counters for the failed operations are never reset since we expect them
- // To be always 0
- return new TestLsmBtreeIoOpCallback(index, getComponentIdGenerator(), getIndexCheckpointManagerProvider());
- }
-
- public int getTotalFlushes() {
- return completedFlushes + rollbackFlushes;
- }
-
- public int getTotalMerges() {
- return completedMerges + rollbackMerges;
- }
-
- public int getTotalIoOps() {
- return getTotalFlushes() + getTotalMerges();
- }
-
- public int getRollbackFlushes() {
- return rollbackFlushes;
- }
-
- public int getRollbackMerges() {
- return rollbackMerges;
- }
-
- public int getCompletedFlushes() {
- return completedFlushes;
- }
-
- public int getCompletedMerges() {
- return completedMerges;
- }
-
- public static int getFailedFlushes() {
- return failedFlushes;
- }
-
- public static int getFailedMerges() {
- return failedMerges;
- }
-
- public class TestLsmBtreeIoOpCallback extends LSMBTreeIOOperationCallback {
- private final TestLsmBtree lsmBtree;
-
- public TestLsmBtreeIoOpCallback(ILSMIndex index, ILSMComponentIdGenerator idGenerator,
- IIndexCheckpointManagerProvider checkpointManagerProvider) {
- super(index, idGenerator, checkpointManagerProvider);
- lsmBtree = (TestLsmBtree) index;
- }
-
- @Override
- public void beforeOperation(ILSMIndexOperationContext opCtx) throws HyracksDataException {
- lsmBtree.beforeIoOperationCalled();
- super.beforeOperation(opCtx);
- lsmBtree.beforeIoOperationReturned();
- }
-
- @Override
- public void afterOperation(ILSMIndexOperationContext opCtx) throws HyracksDataException {
- lsmBtree.afterIoOperationCalled();
- super.afterOperation(opCtx);
- lsmBtree.afterIoOperationReturned();
- }
-
- @Override
- public void afterFinalize(ILSMIndexOperationContext opCtx) throws HyracksDataException {
- lsmBtree.afterIoFinalizeCalled();
- super.afterFinalize(opCtx);
- synchronized (TestLsmBtreeIoOpCallbackFactory.this) {
- if (opCtx.getNewComponent() != null) {
- if (opCtx.getNewComponent() == EmptyComponent.INSTANCE) {
- if (opCtx.getIoOperationType() == LSMIOOperationType.FLUSH) {
- rollbackFlushes++;
- } else {
- rollbackMerges++;
- }
- } else {
- if (opCtx.getIoOperationType() == LSMIOOperationType.FLUSH) {
- completedFlushes++;
- } else {
- completedMerges++;
- }
- }
- } else {
- recordFailure(opCtx.getIoOperationType());
- }
- TestLsmBtreeIoOpCallbackFactory.this.notifyAll();
- }
- lsmBtree.afterIoFinalizeReturned();
- }
-
- @Override
- public void recycled(ILSMMemoryComponent component, boolean advance) throws HyracksDataException {
- lsmBtree.recycledCalled(component);
- super.recycled(component, advance);
- lsmBtree.recycledReturned(component);
- }
-
- @Override
- public void allocated(ILSMMemoryComponent component) throws HyracksDataException {
- lsmBtree.allocatedCalled(component);
- super.allocated(component);
- lsmBtree.allocatedReturned(component);
- }
-
- private void recordFailure(LSMIOOperationType opType) {
- if (opType == LSMIOOperationType.FLUSH) {
- failedFlushes++;
- } else {
- failedMerges++;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
new file mode 100644
index 0000000..c762c8c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestLsmIoOpCallbackFactory.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.dataflow;
+
+import org.apache.asterix.common.api.IDatasetInfoProvider;
+import org.apache.asterix.common.api.ILSMComponentIdGeneratorFactory;
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
+import org.apache.asterix.common.ioopcallbacks.LSMIndexIOOperationCallbackFactory;
+import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.EmptyComponent;
+
+public class TestLsmIoOpCallbackFactory extends LSMIndexIOOperationCallbackFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private static volatile int completedFlushes = 0;
+ private static volatile int completedMerges = 0;
+ private static volatile int rollbackFlushes = 0;
+ private static volatile int rollbackMerges = 0;
+ private static volatile int failedFlushes = 0;
+ private static volatile int failedMerges = 0;
+
+ public TestLsmIoOpCallbackFactory(ILSMComponentIdGeneratorFactory idGeneratorFactory,
+ IDatasetInfoProvider datasetInfoProvider) {
+ super(idGeneratorFactory, datasetInfoProvider);
+ }
+
+ @Override
+ public synchronized ILSMIOOperationCallback createIoOpCallback(ILSMIndex index) throws HyracksDataException {
+ completedFlushes = 0;
+ completedMerges = 0;
+ rollbackFlushes = 0;
+ rollbackMerges = 0;
+ // Whenever this is called, it resets the counter
+ // However, the counters for the failed operations are never reset since we expect them
+ // To be always 0
+ return new TestLsmIoOpCallback(datasetInfoProvider.getDatasetInfo(ncCtx), index,
+ getComponentIdGenerator().getId(), getIndexCheckpointManagerProvider());
+ }
+
+ public int getTotalFlushes() {
+ return completedFlushes + rollbackFlushes;
+ }
+
+ public int getTotalMerges() {
+ return completedMerges + rollbackMerges;
+ }
+
+ public int getTotalIoOps() {
+ return getTotalFlushes() + getTotalMerges();
+ }
+
+ public int getRollbackFlushes() {
+ return rollbackFlushes;
+ }
+
+ public int getRollbackMerges() {
+ return rollbackMerges;
+ }
+
+ public int getCompletedFlushes() {
+ return completedFlushes;
+ }
+
+ public int getCompletedMerges() {
+ return completedMerges;
+ }
+
+ public static int getFailedFlushes() {
+ return failedFlushes;
+ }
+
+ public static int getFailedMerges() {
+ return failedMerges;
+ }
+
+ public class TestLsmIoOpCallback extends LSMIOOperationCallback {
+ private final TestLsmBtree lsmBtree;
+
+ public TestLsmIoOpCallback(DatasetInfo dsInfo, ILSMIndex index, ILSMComponentId id,
+ IIndexCheckpointManagerProvider checkpointManagerProvider) {
+ super(dsInfo, index, id, checkpointManagerProvider);
+ lsmBtree = (TestLsmBtree) index;
+ }
+
+ @Override
+ public void scheduled(ILSMIOOperation op) throws HyracksDataException {
+ lsmBtree.ioScheduledCalled();
+ super.scheduled(op);
+ lsmBtree.ioScheduledReturned();
+ }
+
+ @Override
+ public void beforeOperation(ILSMIOOperation op) throws HyracksDataException {
+ lsmBtree.beforeIoOperationCalled();
+ super.beforeOperation(op);
+ lsmBtree.beforeIoOperationReturned();
+ }
+
+ @Override
+ public void afterOperation(ILSMIOOperation op) throws HyracksDataException {
+ lsmBtree.afterIoOperationCalled();
+ super.afterOperation(op);
+ lsmBtree.afterIoOperationReturned();
+ }
+
+ @Override
+ public void afterFinalize(ILSMIOOperation op) throws HyracksDataException {
+ lsmBtree.afterIoFinalizeCalled();
+ super.afterFinalize(op);
+ synchronized (TestLsmIoOpCallbackFactory.this) {
+ if (op.getNewComponent() != null) {
+ if (op.getNewComponent() == EmptyComponent.INSTANCE) {
+ if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ rollbackFlushes++;
+ } else {
+ rollbackMerges++;
+ }
+ } else {
+ if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) {
+ completedFlushes++;
+ } else {
+ completedMerges++;
+ }
+ }
+ } else {
+ recordFailure(op.getIOOpertionType());
+ }
+ TestLsmIoOpCallbackFactory.this.notifyAll();
+ }
+ lsmBtree.afterIoFinalizeReturned();
+ }
+
+ @Override
+ public void completed(ILSMIOOperation operation) {
+ try {
+ lsmBtree.ioCompletedCalled();
+ super.completed(operation);
+ lsmBtree.ioCompletedReturned();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public void recycled(ILSMMemoryComponent component) throws HyracksDataException {
+ lsmBtree.recycledCalled(component);
+ super.recycled(component);
+ lsmBtree.recycledReturned(component);
+ }
+
+ @Override
+ public void allocated(ILSMMemoryComponent component) throws HyracksDataException {
+ lsmBtree.allocatedCalled(component);
+ super.allocated(component);
+ lsmBtree.allocatedReturned(component);
+ }
+
+ private void recordFailure(LSMIOOperationType opType) {
+ if (opType == LSMIOOperationType.FLUSH) {
+ failedFlushes++;
+ } else {
+ failedMerges++;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
index 9a528d3..250c25f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTracker.java
@@ -60,7 +60,7 @@ public class TestPrimaryIndexOperationTracker extends PrimaryIndexOperationTrack
super.triggerScheduleFlush(logRecord);
synchronized (callbacks) {
for (ITestOpCallback<Void> callback : callbacks) {
- callback.after();
+ callback.after(null);
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
index 06b213d..7a3e475 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TransactionAbortTest.java
@@ -199,7 +199,7 @@ public class TransactionAbortTest {
}
@Override
- public void after() throws HyracksDataException {
+ public void after(Semaphore t) throws HyracksDataException {
// manually set the current memory component as modified
index.getCurrentMemoryComponent().setModified();
throw new HyracksDataException("Fail the job");
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
index 0f6adf6..d5985b5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/metadata/MetadataTxnTest.java
@@ -51,7 +51,6 @@ import org.apache.hyracks.api.util.InvokeUtil;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
-import org.apache.hyracks.test.support.TestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -261,8 +260,9 @@ public class MetadataTxnTest {
final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxn);
final String nodeGroupName = "ng";
+ final String committedNodeGroup = "committed_ng";
+ final List<String> ngNodes = Collections.singletonList("asterix_nc1");
try {
- final List<String> ngNodes = Collections.singletonList("asterix_nc1");
MetadataManager.INSTANCE.addNodegroup(mdTxn, new NodeGroup(nodeGroupName, ngNodes));
MetadataManager.INSTANCE.commitTransaction(mdTxn);
} finally {
@@ -281,6 +281,9 @@ public class MetadataTxnTest {
int diskComponentsBeforeFlush = index.getDiskComponents().size();
// lock opTracker to prevent log flusher from triggering flush
synchronized (opTracker) {
+ final MetadataTransactionContext committedMdTxn = MetadataManager.INSTANCE.beginTransaction();
+ MetadataManager.INSTANCE.addNodegroup(committedMdTxn, new NodeGroup(committedNodeGroup, ngNodes));
+ MetadataManager.INSTANCE.commitTransaction(committedMdTxn);
opTracker.setFlushOnExit(true);
opTracker.flushIfNeeded();
Assert.assertTrue(opTracker.isFlushLogCreated());
@@ -288,7 +291,6 @@ public class MetadataTxnTest {
// make sure force operation will processed
MetadataManager.INSTANCE.dropNodegroup(mdTxn2, nodeGroupName, false);
Assert.assertEquals(1, opTracker.getNumActiveOperations());
- Assert.assertFalse(index.hasFlushRequestForCurrentMutableComponent());
// release opTracker lock now to allow log flusher to schedule the flush
InvokeUtil.runWithTimeout(() -> {
synchronized (opTracker) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
deleted file mode 100644
index 17509a4..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/DiskIsFullTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.storage;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
-import org.apache.asterix.app.bootstrap.TestNodeController;
-import org.apache.asterix.app.data.gen.TupleGenerator;
-import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction;
-import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable;
-import org.apache.asterix.common.exceptions.ExceptionUtils;
-import org.apache.asterix.common.transactions.ITransactionContext;
-import org.apache.asterix.common.transactions.ITransactionManager;
-import org.apache.asterix.common.transactions.TransactionOptions;
-import org.apache.asterix.external.util.DataflowUtils;
-import org.apache.asterix.file.StorageComponentProvider;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.test.common.TestHelper;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
-import org.apache.hyracks.util.DiskUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class DiskIsFullTest {
-
- private static final IAType[] KEY_TYPES = { BuiltinType.AINT32 };
- private static final ARecordType RECORD_TYPE = new ARecordType("TestRecordType", new String[] { "key", "value" },
- new IAType[] { BuiltinType.AINT32, BuiltinType.AINT64 }, false);
- private static final GenerationFunction[] RECORD_GEN_FUNCTION =
- { GenerationFunction.DETERMINISTIC, GenerationFunction.DETERMINISTIC };
- private static final boolean[] UNIQUE_RECORD_FIELDS = { true, false };
- private static final ARecordType META_TYPE = null;
- private static final GenerationFunction[] META_GEN_FUNCTION = null;
- private static final boolean[] UNIQUE_META_FIELDS = null;
- private static final int[] KEY_INDEXES = { 0 };
- private static final int[] KEY_INDICATOR = { Index.RECORD_INDICATOR };
- private static final List<Integer> KEY_INDICATOR_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR });
- private static final int DATASET_ID = 101;
- private static final String DATAVERSE_NAME = "TestDV";
- private static final String DATASET_NAME = "TestDS";
- private static final String DATA_TYPE_NAME = "DUMMY";
- private static final String NODE_GROUP_NAME = "DEFAULT";
- private static final String TEST_DISK_NAME = "asterixdb_ram_disk";
- private boolean shouldRun = true;
-
- @Before
- public void setUp() throws Exception {
- if (!SystemUtils.IS_OS_MAC) {
- System.out.println("Skipping test " + DiskIsFullTest.class.getName() + " due to unsupported OS");
- shouldRun = false;
- return;
- }
- System.out.println("SetUp: ");
- TestHelper.deleteExistingInstanceFiles();
- // create RAM disk
- final Path ramDiskRoot = DiskUtil.mountRamDisk(TEST_DISK_NAME, 4, MEGABYTE);
- // Use RAM disk for storage
- AsterixHyracksIntegrationUtil.setStoragePath(ramDiskRoot.toAbsolutePath().toString());
- }
-
- @After
- public void tearDown() throws Exception {
- if (!shouldRun) {
- return;
- }
- System.out.println("TearDown");
- TestHelper.deleteExistingInstanceFiles();
- DiskUtil.unmountRamDisk(TEST_DISK_NAME);
- AsterixHyracksIntegrationUtil.restoreDefaultStoragePath();
- }
-
- @Test
- public void testDiskIsFull() {
- if (!shouldRun) {
- return;
- }
- HyracksDataException expectedException =
- HyracksDataException.create(ErrorCode.CANNOT_MODIFY_INDEX_DISK_IS_FULL);
- try {
- TestNodeController nc = new TestNodeController(null, false);
- nc.init();
- StorageComponentProvider storageManager = new StorageComponentProvider();
- List<List<String>> partitioningKeys = new ArrayList<>();
- partitioningKeys.add(Collections.singletonList("key"));
- Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME,
- NoMergePolicyFactory.NAME, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH,
- partitioningKeys, null, null, null, false, null),
- null, DatasetType.INTERNAL, DATASET_ID, 0);
- try {
- nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, null, storageManager, KEY_INDEXES,
- KEY_INDICATOR_LIST, 0);
- JobId jobId = nc.newJobId();
- IHyracksTaskContext ctx = nc.createTestContext(jobId, 0, false);
- ITransactionContext txnCtx = nc.getTransactionManager().beginTransaction(nc.getTxnJobId(ctx),
- new TransactionOptions(ITransactionManager.AtomicityLevel.ENTITY_LEVEL));
- // Prepare insert operation
- LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES,
- RECORD_TYPE, META_TYPE, null, KEY_INDEXES, KEY_INDICATOR_LIST, storageManager, null).getLeft();
- try {
- insertOp.open();
- TupleGenerator tupleGenerator =
- new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATOR, RECORD_GEN_FUNCTION,
- UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
- VSizeFrame frame = new VSizeFrame(ctx);
- FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
- // Insert records until disk becomes full
- int tupleCount = 100000;
- while (tupleCount > 0) {
- ITupleReference tuple = tupleGenerator.next();
- try {
- DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
- } catch (Throwable t) {
- final Throwable rootCause = ExceptionUtils.getRootCause(t);
- rootCause.printStackTrace();
- if (rootCause instanceof HyracksDataException) {
- HyracksDataException cause = (HyracksDataException) rootCause;
- Assert.assertEquals(cause.getErrorCode(), expectedException.getErrorCode());
- Assert.assertEquals(cause.getMessage(), expectedException.getMessage());
- return;
- } else {
- break;
- }
- }
- tupleCount--;
- }
- Assert.fail("Expected exception (" + expectedException + ") was not thrown");
- } finally {
- try {
- insertOp.close();
- } finally {
- nc.getTransactionManager().abortTransaction(txnCtx.getTxnId());
- }
- }
- } finally {
- nc.deInit();
- }
- } catch (Throwable e) {
- e.printStackTrace();
- Assert.fail("Expected exception (" + expectedException + ") was not thrown");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
new file mode 100644
index 0000000..7075739
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IDatasetInfoProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.hyracks.api.application.INCServiceContext;
+
+@FunctionalInterface
+public interface IDatasetInfoProvider extends Serializable {
+ /**
+ * Get the dataset info
+ *
+ * @param serviceCtx
+ * the nc service context
+ * @return the dataset info object
+ */
+ DatasetInfo getDatasetInfo(INCServiceContext serviceCtx);
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
new file mode 100644
index 0000000..5fd37e9
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ILSMComponentIdGeneratorFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator;
+import org.apache.hyracks.storage.common.IResource;
+
+@FunctionalInterface
+public interface ILSMComponentIdGeneratorFactory extends Serializable {
+ /**
+ * Get the Id generator for the resource
+ *
+ * @param serviceCtx
+ * @param resource
+ * @return
+ * @throws HyracksDataException
+ */
+ ILSMComponentIdGenerator getComponentIdGenerator(INCServiceContext serviceCtx, IResource resource)
+ throws HyracksDataException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/4ed0eac5/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index b7af0b6..3ff13cb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -37,8 +37,7 @@ public class BaseOperationTracker implements ITransactionOperationTracker {
@Override
public void beforeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
- if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE
- || opType == LSMOperationType.REPLICATE) {
+ if (opType == LSMOperationType.REPLICATE) {
dsInfo.declareActiveIOOperation();
}
}
@@ -47,14 +46,14 @@ public class BaseOperationTracker implements ITransactionOperationTracker {
public void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
if (opType == LSMOperationType.REPLICATE) {
- dsInfo.undeclareActiveIOOperation();
+ completeOperation(index, opType, searchCallback, modificationCallback);
}
}
@Override
public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
- if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE) {
+ if (opType == LSMOperationType.REPLICATE) {
dsInfo.undeclareActiveIOOperation();
}
}