You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ti...@apache.org on 2018/11/18 17:48:41 UTC
asterixdb git commit: [NO ISSUE][STO] Let ILSMIOOperations clean up
for themselves
Repository: asterixdb
Updated Branches:
refs/heads/master ac1b93529 -> b4f663115
[NO ISSUE][STO] Let ILSMIOOperations clean up for themselves
- user model changes: no
- storage format changes: no
- interface changes: yes
remove ILSMIndex.cleanUpFilesForFailedOperation
remove ILSMIOOperation.getComponentFiles
add ILSMIOOperation.cleanup
Details:
Instead of having ILSMIndex instances clean up for ILSMIOOperations, let
the ILSMIOOperations cleanup their component files.
Also avoids a dependency of the interface ILSMIOOperation on
o.a.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences.
Change-Id: I97a31d018139dcdea15196ae52d91eed5e780fbd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3034
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b4f66311
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b4f66311
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b4f66311
Branch: refs/heads/master
Commit: b4f66311584ddb42470d0e185a477282f4aaf73c
Parents: ac1b935
Author: Till Westmann <ti...@apache.org>
Authored: Sat Nov 17 19:47:47 2018 -0800
Committer: Till Westmann <ti...@apache.org>
Committed: Sun Nov 18 09:48:11 2018 -0800
----------------------------------------------------------------------
.../am/lsm/common/api/ILSMIOOperation.java | 9 +++++---
.../storage/am/lsm/common/api/ILSMIndex.java | 7 ------
.../lsm/common/impls/AbstractIoOperation.java | 23 ++++++++++++++++++++
.../am/lsm/common/impls/AbstractLSMIndex.java | 19 ----------------
.../storage/am/lsm/common/impls/LSMHarness.java | 2 +-
.../impls/LSMIndexDiskComponentBulkLoader.java | 2 +-
.../am/lsm/common/impls/NoOpIoOperation.java | 3 ++-
.../am/lsm/common/impls/TracedIOOperation.java | 5 +++--
8 files changed, 36 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
index 0e13933..79463e5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIOOperation.java
@@ -25,7 +25,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationStatus;
-import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.IPageWriteFailureCallback;
public interface ILSMIOOperation extends Callable<LSMIOOperationStatus>, IPageWriteFailureCallback {
@@ -88,9 +88,12 @@ public interface ILSMIOOperation extends Callable<LSMIOOperationStatus>, IPageWr
ILSMIndexAccessor getAccessor();
/**
- * @return the component files produced by this operation
+ * clean up left over files in case of an exception during execution
+ *
+ * @param bufferCache
+ * a buffercache that manages the files
*/
- LSMComponentFileReferences getComponentFiles();
+ void cleanup(IBufferCache bufferCache);
/**
* @return the failure in the io operation if any, null otherwise
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index 5cb05a7..23b3634 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -181,13 +181,6 @@ public interface ILSMIndex extends IIndex {
ILSMHarness getHarness();
/**
- * Cleanup the files of the failed operation
- *
- * @param operation
- */
- void cleanUpFilesForFailedOperation(ILSMIOOperation operation);
-
- /**
* @return the absolute path of the index
*/
String getIndexIdentifier();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
index 3d76755..38ef179 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractIoOperation.java
@@ -22,6 +22,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.util.ExceptionUtils;
@@ -30,6 +31,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public abstract class AbstractIoOperation implements ILSMIOOperation {
@@ -78,6 +80,27 @@ public abstract class AbstractIoOperation implements ILSMIOOperation {
}
@Override
+ public void cleanup(IBufferCache bufferCache) {
+ LSMComponentFileReferences componentFiles = getComponentFiles();
+ if (componentFiles == null) {
+ return;
+ }
+ FileReference[] files = componentFiles.getFileReferences();
+ for (FileReference file : files) {
+ try {
+ if (file != null) {
+ bufferCache.closeFileIfOpen(file);
+ bufferCache.deleteFile(file);
+ }
+ } catch (HyracksDataException hde) {
+ getFailure().addSuppressed(hde);
+ }
+ }
+ }
+
+ protected abstract LSMComponentFileReferences getComponentFiles();
+
+ @Override
public Throwable getFailure() {
return failure;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index d3133ce..cf80472 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -860,25 +860,6 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
}
@Override
- public void cleanUpFilesForFailedOperation(ILSMIOOperation operation) {
- LSMComponentFileReferences componentFiles = operation.getComponentFiles();
- if (componentFiles == null) {
- return;
- }
- FileReference[] files = componentFiles.getFileReferences();
- for (FileReference file : files) {
- try {
- if (file != null) {
- diskBufferCache.closeFileIfOpen(file);
- diskBufferCache.deleteFile(file);
- }
- } catch (Throwable th) { // NOSONAR Must catch all failures
- operation.getFailure().addSuppressed(th);
- }
- }
- }
-
- @Override
public String getIndexIdentifier() {
return fileManager.getBaseDir().getAbsolutePath();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index e9f6f20..dc54e20 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -553,7 +553,7 @@ public class LSMHarness implements ILSMHarness {
}
// if the operation failed, we need to cleanup files
if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
- lsmIndex.cleanUpFilesForFailedOperation(operation);
+ operation.cleanup(lsmIndex.getBufferCache());
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
index c739ad0..9b36383 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexDiskComponentBulkLoader.java
@@ -129,7 +129,7 @@ public class LSMIndexDiskComponentBulkLoader implements IIndexBulkLoader {
failed = true;
final ILSMIOOperation loadOp = opCtx.getIoOperation();
loadOp.setFailure(th);
- lsmIndex.cleanUpFilesForFailedOperation(loadOp);
+ loadOp.cleanup(lsmIndex.getBufferCache());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
index 5ee1503..cf0f4e5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/NoOpIoOperation.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
public class NoOpIoOperation implements ILSMIOOperation {
@@ -73,7 +74,7 @@ public class NoOpIoOperation implements ILSMIOOperation {
}
@Override
- public LSMComponentFileReferences getComponentFiles() {
+ public void cleanup(IBufferCache bufferCache) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b4f66311/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
index 3345e3a..a2304ed 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
@@ -29,6 +29,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.storage.am.lsm.common.api.IoOperationCompleteListener;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.ITracer.Scope;
@@ -110,8 +111,8 @@ class TracedIOOperation implements ILSMIOOperation {
}
@Override
- public LSMComponentFileReferences getComponentFiles() {
- return ioOp.getComponentFiles();
+ public void cleanup(IBufferCache bufferCache) {
+ ioOp.cleanup(bufferCache);
}
@Override