You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2017/08/12 22:15:55 UTC
asterixdb git commit: [ASTERIXDB-2025][STO] Fix Merge Lifecycle
Repository: asterixdb
Updated Branches:
refs/heads/master 4cf1f7b97 -> 476f14c38
[ASTERIXDB-2025][STO] Fix Merge Lifecycle
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Complete merge operation after deletion of old components
files.
Change-Id: I843de8b26c181205e43f4eabe22a7c43f3ebfcbc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1930
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/476f14c3
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/476f14c3
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/476f14c3
Branch: refs/heads/master
Commit: 476f14c38bc551602cee97d26bc55fd4546927d6
Parents: 4cf1f7b
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Fri Aug 11 10:39:14 2017 -0700
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Sat Aug 12 15:15:06 2017 -0700
----------------------------------------------------------------------
.../common/context/BaseOperationTracker.java | 6 +++--
.../common/context/DatasetLifecycleManager.java | 3 ++-
.../context/PrimaryIndexOperationTracker.java | 3 +--
.../PersistentLocalResourceRepository.java | 3 ++-
.../am/common/dataflow/IndexDataflowHelper.java | 5 ++++
.../storage/am/lsm/common/impls/LSMHarness.java | 28 +++++++++++++++++---
.../lsm/common/impls/LSMTreeIndexAccessor.java | 5 ++++
.../impls/LSMInvertedIndexAccessor.java | 4 +++
8 files changed, 47 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index 9cb1de5..e74600e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -47,8 +47,7 @@ public class BaseOperationTracker implements ILSMOperationTracker {
@Override
public void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
- if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE
- || opType == LSMOperationType.REPLICATE) {
+ if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.REPLICATE) {
dsInfo.undeclareActiveIOOperation();
}
}
@@ -56,6 +55,9 @@ public class BaseOperationTracker implements ILSMOperationTracker {
@Override
public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
+ if (opType == LSMOperationType.MERGE) {
+ dsInfo.undeclareActiveIOOperation();
+ }
}
public void exclusiveJobCommitted() throws HyracksDataException {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 8abbeab..37bd789 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -147,7 +147,8 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC
//notification will come from DatasetInfo class (undeclareActiveIOOperation)
dsInfo.wait();
} catch (InterruptedException e) {
- throw new HyracksDataException(e);
+ Thread.currentThread().interrupt();
+ throw HyracksDataException.create(e);
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
index 903bb50..67b25b6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/PrimaryIndexOperationTracker.java
@@ -68,8 +68,7 @@ public class PrimaryIndexOperationTracker extends BaseOperationTracker {
public void afterOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
// Searches are immediately considered complete, because they should not prevent the execution of flushes.
- if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.MERGE
- || opType == LSMOperationType.REPLICATE) {
+ if (opType == LSMOperationType.FLUSH || opType == LSMOperationType.REPLICATE) {
completeOperation(index, opType, searchCallback, modificationCallback);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index b117cf1..85ad6b4 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -59,6 +59,7 @@ import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationJobType;
import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.LocalResource;
@@ -223,7 +224,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
// Invalidate before deleting the file just in case file deletion throws some exception.
// Since it's just a cache invalidation, it should not affect correctness.
resourceCache.invalidate(relativePath);
- resourceFile.delete();
+ IoUtil.delete(resourceFile);
} finally {
// Regardless of successfully deleted or not, the operation should be replicated.
//if replication enabled, delete resource from remote replicas
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
index 1e959e1..af843d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -19,6 +19,9 @@
package org.apache.hyracks.storage.am.common.dataflow;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -32,6 +35,7 @@ import org.apache.hyracks.storage.common.LocalResource;
public class IndexDataflowHelper implements IIndexDataflowHelper {
+ private static final Logger LOGGER = Logger.getLogger(IndexDataflowHelper.class.getName());
private final INCServiceContext ctx;
private final IResourceLifecycleManager<IIndex> lcManager;
private final ILocalResourceRepository localResourceRepository;
@@ -85,6 +89,7 @@ public class IndexDataflowHelper implements IIndexDataflowHelper {
@Override
public void destroy() throws HyracksDataException {
+ LOGGER.log(Level.INFO, "Dropping index " + resourceRef.getRelativePath() + " on node " + ctx.getNodeId());
synchronized (lcManager) {
index = lcManager.get(resourceRef.getRelativePath());
if (index != null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index b1005bd..d0dc4b3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -138,7 +138,8 @@ public class LSMHarness implements ILSMHarness {
}
opTracker.wait();
} catch (InterruptedException e) {
- throw new HyracksDataException(e);
+ Thread.currentThread().interrupt();
+ throw HyracksDataException.create(e);
}
}
}
@@ -346,12 +347,11 @@ public class LSMHarness implements ILSMHarness {
lsmIndex.scheduleReplication(null, inactiveDiskComponentsToBeDeleted, false,
ReplicationOperation.DELETE, opType);
}
-
for (ILSMComponent c : inactiveDiskComponentsToBeDeleted) {
((AbstractLSMDiskComponent) c).destroy();
}
} catch (Throwable e) {
- e.printStackTrace();
+ LOGGER.log(Level.WARNING, "Failure scheduling replication or destroying merged component", e);
throw e;
}
}
@@ -561,10 +561,25 @@ public class LSMHarness implements ILSMHarness {
lsmIndex.markAsValid(newComponent);
} catch (Throwable e) {
failedOperation = true;
- e.printStackTrace();
+ LOGGER.log(Level.SEVERE, "Failed merge operation on " + lsmIndex, e);
throw e;
} finally {
exitComponents(ctx, LSMOperationType.MERGE, newComponent, failedOperation);
+ // Completion of the merge operation is called here to and not on afterOperation because
+ // Deletion of the old components comes after afterOperation is called and the number of
+ // io operation should not be decremented before the operation is complete to avoid
+ // index destroy from competing with the merge on deletion of the files.
+ // The order becomes:
+ // 1. scheduleMerge
+ // 2. enterComponents
+ // 3. beforeOperation (increment the numOfIoOperations)
+ // 4. merge
+ // 5. exitComponents
+ // 6. afterOperation (no op)
+ // 7. delete components
+ // 8. completeOperation (decrement the numOfIoOperations)
+ opTracker.completeOperation(lsmIndex, LSMOperationType.MERGE, ctx.getSearchOperationCallback(),
+ ctx.getModificationCallback());
operation.getCallback().afterFinalize(LSMOperationType.MERGE, newComponent);
}
if (LOGGER.isLoggable(Level.INFO)) {
@@ -701,4 +716,9 @@ public class LSMHarness implements ILSMHarness {
}
throw HyracksDataException.create(ErrorCode.CANNOT_MODIFY_INDEX_DISK_IS_FULL);
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ":" + lsmIndex;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index b9714a6..a45225d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -219,4 +219,9 @@ public class LSMTreeIndexAccessor implements ILSMIndexAccessor {
ctx.setOperation(IndexOperation.DISK_COMPONENT_SCAN);
lsmHarness.scanDiskComponents(ctx, cursor);
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ':' + lsmHarness.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/476f14c3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index 7751116..dddd14a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -207,6 +207,10 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedInd
@Override
public void scanDiskComponents(IIndexCursor cursor) throws HyracksDataException {
throw HyracksDataException.create(ErrorCode.DISK_COMPONENT_SCAN_NOT_ALLOWED_FOR_SECONDARY_INDEX);
+ }
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ':' + lsmHarness.toString();
}
}